diff options
145 files changed, 5335 insertions, 35152 deletions
diff --git a/Makefile.in b/Makefile.in index 995bbde44..d462d811e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -264,7 +264,9 @@ AS_AUTO_DEF = $(TOP)/auto.def # invoked with to produce this makefile. # AS_AUTORECONFIG = @SQLITE_AUTORECONFIG@ - +.PHONY: reconfigure +reconfigure: + $(AS_AUTORECONFIG) USE_AMALGAMATION ?= @USE_AMALGAMATION@ LINK_TOOLS_DYNAMICALLY ?= @LINK_TOOLS_DYNAMICALLY@ AMALGAMATION_GEN_FLAGS ?= --linemacros=@AMALGAMATION_LINE_MACROS@ diff --git a/Makefile.msc b/Makefile.msc index 1533cf3de..e67638287 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2596,9 +2596,12 @@ extensiontest: testfixture.exe testloadext.dll @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS) -tool-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe $(TOP)\tool\mktoolzip.tcl +tool-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe sqlite3.dll $(TOP)\tool\mktoolzip.tcl .\testfixture.exe $(TOP)\tool\mktoolzip.tcl +snapshot-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe sqlite3.dll $(TOP)\tool\mktoolzip.tcl + .\testfixture.exe $(TOP)\tool\mktoolzip.tcl --snapshot + coretestprogs: testfixture.exe sqlite3.exe testprogs: $(TESTPROGS) srcck1.exe fuzzcheck.exe sessionfuzz.exe diff --git a/autoconf/tea/configure b/autoconf/tea/configure index 47378126f..01b3abcc2 100755 --- a/autoconf/tea/configure +++ b/autoconf/tea/configure @@ -1,7 +1,20 @@ #!/bin/sh +# Look for and run autosetup... dir0="`dirname "$0"`" -dirA="$dir0/../autosetup" -# This is the case ^^^^^^^^^^^^ in the SQLite "autoconf" bundle. +dirA="$dir0" +if [ -d $dirA/autosetup ]; then + # A local copy of autosetup + dirA=$dirA/autosetup +elif [ -d $dirA/../autosetup ]; then + # SQLite "autoconf" bundle + dirA=$dirA/../autosetup +elif [ -d $dirA/../../autosetup ]; then + # SQLite canonical source tree + dirA=$dirA/../../autosetup +else + echo "$0: Cannot find autosetup" 1>&2 + exit 1 +fi WRAPPER="$0"; export WRAPPER; exec "`"$dirA/autosetup-find-tclsh"`" \ "$dirA/autosetup" --teaish-extension-dir="$dir0" \ "$@" diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 5cebc0472..fca85e9ad 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1604,7 +1604,7 @@ proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { # # Similar modifications may be made for --mandir. # -# Returns 1 if it modifies the environment, else 0. +# Returns >0 if it modifies the environment, else 0. # proc proj-tweak-default-env-dirs {} { set rc 0 diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index bb8112320..4dd065095 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -336,8 +336,8 @@ proc sqlite-configure {buildMode configScript} { => {Link the sqlite3 shell app against the DLL instead of embedding sqlite3.c} } {canonical autoconf} { - # A potential TODO without a current use case: - #rpath=1 => {Disable use of the rpath linker flag} + rpath=1 => {Disable use of the rpath linker flag} + # soname: https://sqlite.org/src/forumpost/5a3b44f510df8ded soname:=legacy => {SONAME for libsqlite3.so. "none", or not using this flag, sets no @@ -2140,7 +2140,6 @@ proc sqlite-handle-tcl {} { ######################################################################## # Handle the --enable/disable-rpath flag. proc sqlite-handle-rpath {} { - proj-check-rpath # autosetup/cc-shared.tcl sets the rpath flag definition in # [get-define SH_LINKRPATH], but it does so on a per-platform basis # rather than as a compiler check. Though we should do a proper @@ -2149,12 +2148,13 @@ proc sqlite-handle-rpath {} { # for which sqlite-env-is-unix-on-windows returns a non-empty # string. -# if {[proj-opt-truthy rpath]} { -# proj-check-rpath -# } else { -# msg-result "Disabling use of rpath." -# define LDFLAGS_RPATH "" -# } + # https://sqlite.org/forum/forumpost/13cac3b56516f849 + if {[proj-opt-truthy rpath]} { + proj-check-rpath + } else { + msg-result "Disabling use of rpath." + define LDFLAGS_RPATH "" + } } ######################################################################## diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 63840de1f..6345352c5 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -554,6 +554,36 @@ struct Fts5SegIter { u8 bDel; /* True if the delete flag is set */ }; +static int fts5IndexCorruptRowid(Fts5Index *pIdx, i64 iRowid){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption found reading blob %lld from table \"%s\"", + iRowid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ROWID(pIdx, iRowid) fts5IndexCorruptRowid(pIdx, iRowid) + +static int fts5IndexCorruptIter(Fts5Index *pIdx, Fts5SegIter *pIter){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption on page %d, segment %d, table \"%s\"", + pIter->iLeafPgno, pIter->pSeg->iSegid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ITER(pIdx, pIter) fts5IndexCorruptIter(pIdx, pIter) + +static int fts5IndexCorruptIdx(Fts5Index *pIdx){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption in table \"%s\"", pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_IDX(pIdx) fts5IndexCorruptIdx(pIdx) + + /* ** Array of tombstone pages. Reference counted. */ @@ -843,7 +873,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ ** All the reasons those functions might return SQLITE_ERROR - missing ** table, missing row, non-blob/text in block column - indicate ** backing store corruption. */ - if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT_ROWID(p, iRowid); if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ @@ -893,7 +923,7 @@ static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = fts5DataRead(p, iRowid); if( pRet ){ if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); fts5DataRelease(pRet); pRet = 0; } @@ -1252,8 +1282,14 @@ static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ - p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + if( p->rc==SQLITE_OK ){ + if( (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + }else if( p->rc==SQLITE_CORRUPT_VTAB ){ + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: corrupt structure record for table \"%s\"", p->pConfig->zName + ); } fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ @@ -1876,7 +1912,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_ITER(p, pIter); return; } iOff = 4; @@ -1908,7 +1944,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ iOff += fts5GetVarint32(&a[iOff], nNew); if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->term.n = nKeep; @@ -1950,9 +1986,9 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ ** leave an error in the Fts5Index object. */ static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ - const int nTomb = pIter->pSeg->nPgTombstone; + const i64 nTomb = (i64)pIter->pSeg->nPgTombstone; if( nTomb>0 ){ - int nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1); + i64 nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1); Fts5TombstoneArray *pNew; pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -2103,7 +2139,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ iRowidOff = fts5LeafFirstRowidOff(pNew); if( iRowidOff ){ if( iRowidOff>=pNew->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); }else{ pIter->pLeaf = pNew; pIter->iLeafOffset = iRowidOff; @@ -2337,7 +2373,7 @@ static void fts5SegIterNext( } assert_nc( iOff<pLeaf->szLeaf ); if( iOff>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } } @@ -2445,18 +2481,20 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ fts5DataRelease(pIter->pLeaf); pIter->pLeaf = pLast; pIter->iLeafPgno = pgnoLast; - iOff = fts5LeafFirstRowidOff(pLast); - if( iOff>pLast->szLeaf ){ - p->rc = FTS5_CORRUPT; - return; - } - iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; + if( p->rc==SQLITE_OK ){ + iOff = fts5LeafFirstRowidOff(pLast); + if( iOff>pLast->szLeaf ){ + FTS5_CORRUPT_ITER(p, pIter); + return; + } + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; - if( fts5LeafIsTermless(pLast) ){ - pIter->iEndofDoclist = pLast->nn+1; - }else{ - pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } } } @@ -2526,7 +2564,7 @@ static void fts5LeafSeek( iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; if( iOff>n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -2569,7 +2607,7 @@ static void fts5LeafSeek( iOff = iTermOff; if( iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -2591,7 +2629,7 @@ static void fts5LeafSeek( iPgidx = (u32)pIter->pLeaf->szLeaf; iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; }else{ nKeep = 0; @@ -2606,7 +2644,7 @@ static void fts5LeafSeek( search_success: if( (i64)iOff+nNew>n || nNew<1 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->iLeafOffset = iOff + nNew; @@ -3071,7 +3109,7 @@ static void fts5SegIterGotoPage( assert( iLeafPgno>pIter->iLeafPgno ); if( iLeafPgno>pIter->pSeg->pgnoLast ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; @@ -3086,7 +3124,7 @@ static void fts5SegIterGotoPage( u8 *a = pIter->pLeaf->p; int n = pIter->pLeaf->szLeaf; if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; @@ -3565,7 +3603,7 @@ static void fts5ChunkIterate( if( nRem<=0 ){ break; }else if( pSeg->pSeg==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); return; }else{ pgno++; @@ -4668,7 +4706,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ ** a single page has been assigned to more than one segment. In ** this case a prior iteration of this loop may have corrupted the ** segment currently being trimmed. */ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iLeafRowid); }else{ fts5BufferZero(&buf); fts5BufferGrow(&p->rc, &buf, pData->nn); @@ -5135,7 +5173,7 @@ static void fts5SecureDeleteOverflow( }else if( bDetailNone ){ break; }else if( iNext>=pLeaf->szLeaf || pLeaf->nn<pLeaf->szLeaf || iNext<4 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); break; }else{ int nShift = iNext - 4; @@ -5155,7 +5193,7 @@ static void fts5SecureDeleteOverflow( i1 += fts5GetVarint32(&aPg[i1], iFirst); if( iFirst<iNext ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); break; } aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); @@ -5378,14 +5416,14 @@ static void fts5DoSecureDelete( nSuffix = (nPrefix2 + nSuffix2) - nPrefix; if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ if( iKey!=1 ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); if( nPrefix2>pSeg->term.n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); iOff += (nPrefix2-nPrefix); @@ -5809,7 +5847,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( } nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel); - assert( nByte==SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); + assert( nByte==(i64)SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -6178,7 +6216,7 @@ static void fts5MergePrefixLists( } if( pHead==0 || pHead->pNext==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); break; } @@ -6215,7 +6253,7 @@ static void fts5MergePrefixLists( assert_nc( tmp.n+nTail<=nTmp ); assert( tmp.n+nTail<=nTmp+nMerge*10 ); if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_IDX(p); break; } fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2); @@ -6784,10 +6822,13 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure *pTmp; - u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; + union { + Fts5Structure sFts; + u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; + } uFts; fts5StructureInvalidate(p); fts5IndexDiscardData(p); - pTmp = (Fts5Structure*)tmpSpace; + pTmp = &uFts.sFts; memset(pTmp, 0, SZ_FTS5STRUCTURE(1)); if( p->pConfig->bContentlessDelete ){ pTmp->nOriginCntr = 1; @@ -8248,19 +8289,27 @@ static int fts5TestUtf8(const char *z, int n){ /* ** This function is also purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. +** +** This function sets output variable (*pbFail) to true if the test fails. Or +** leaves it unchanged if the test succeeds. */ static void fts5TestTerm( Fts5Index *p, Fts5Buffer *pPrev, /* Previous term */ const char *z, int n, /* Possibly new term to test */ u64 expected, - u64 *pCksum + u64 *pCksum, + int *pbFail ){ int rc = p->rc; if( pPrev->n==0 ){ fts5BufferSet(&rc, pPrev, n, (const u8*)z); }else - if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ + if( *pbFail==0 + && rc==SQLITE_OK + && (pPrev->n!=n || memcmp(pPrev->p, z, n)) + && (p->pHash==0 || p->pHash->nEntry==0) + ){ u64 cksum3 = *pCksum; const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ @@ -8310,7 +8359,7 @@ static void fts5TestTerm( fts5BufferSet(&rc, pPrev, n, (const u8*)z); if( rc==SQLITE_OK && cksum3!=expected ){ - rc = FTS5_CORRUPT; + *pbFail = 1; } *pCksum = cksum3; } @@ -8319,7 +8368,7 @@ static void fts5TestTerm( #else # define fts5TestDlidxReverse(x,y,z) -# define fts5TestTerm(u,v,w,x,y,z) +# define fts5TestTerm(t,u,v,w,x,y,z) #endif /* @@ -8344,14 +8393,17 @@ static void fts5IndexIntegrityCheckEmpty( for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); if( pLeaf ){ - if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; - if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; + if( !fts5LeafIsTermless(pLeaf) + || (i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf)) + ){ + FTS5_CORRUPT_ROWID(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); + } } fts5DataRelease(pLeaf); } } -static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ +static void fts5IntegrityCheckPgidx(Fts5Index *p, i64 iRowid, Fts5Data *pLeaf){ i64 iTermOff = 0; int ii; @@ -8369,12 +8421,12 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff = iTermOff; if( iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else if( iTermOff==nIncr ){ int nByte; iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); } @@ -8383,7 +8435,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ buf1.n = nKeep; fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); @@ -8391,7 +8443,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ if( p->rc==SQLITE_OK ){ res = fts5BufferCompare(&buf1, &buf2); - if( res<=0 ) p->rc = FTS5_CORRUPT; + if( res<=0 ) FTS5_CORRUPT_ROWID(p, iRowid); } } fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); @@ -8452,7 +8504,7 @@ static void fts5IndexIntegrityCheckSegment( ** entry even if all the terms are removed from it by secure-delete ** operations. */ }else{ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); } }else{ @@ -8464,15 +8516,15 @@ static void fts5IndexIntegrityCheckSegment( iOff = fts5LeafFirstTermOff(pLeaf); iRowidOff = fts5LeafFirstRowidOff(pLeaf); if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); if( res==0 ) res = nTerm - nIdxTerm; - if( res<0 ) p->rc = FTS5_CORRUPT; + if( res<0 ) FTS5_CORRUPT_ROWID(p, iRow); } - fts5IntegrityCheckPgidx(p, pLeaf); + fts5IntegrityCheckPgidx(p, iRow, pLeaf); } fts5DataRelease(pLeaf); if( p->rc ) break; @@ -8502,7 +8554,7 @@ static void fts5IndexIntegrityCheckSegment( iKey = FTS5_SEGMENT_ROWID(iSegid, iPg); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ - if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) FTS5_CORRUPT_ROWID(p, iKey); fts5DataRelease(pLeaf); } } @@ -8517,12 +8569,12 @@ static void fts5IndexIntegrityCheckSegment( int iRowidOff = fts5LeafFirstRowidOff(pLeaf); ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); }else if( bSecureDelete==0 || iRowidOff>0 ){ i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); } } fts5DataRelease(pLeaf); @@ -8574,6 +8626,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ + int bTestFail = 0; #endif const int flags = FTS5INDEX_QUERY_NOOUTPUT; @@ -8616,7 +8669,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ - fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + fts5TestTerm(p, &term, z, n, cksum2, &cksum3, &bTestFail); if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ @@ -8634,15 +8687,26 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ } } } - fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3, &bTestFail); fts5MultiIterFree(pIter); - if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; - - fts5StructureRelease(pStruct); + if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ){ + p->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: checksum mismatch for table \"%s\"", p->pConfig->zName + ); + } #ifdef SQLITE_DEBUG + /* In SQLITE_DEBUG builds, expensive extra checks were run as part of + ** the integrity-check above. If no other errors were detected, but one + ** of these tests failed, set the result to SQLITE_CORRUPT_VTAB here. */ + if( p->rc==SQLITE_OK && bTestFail ){ + p->rc = FTS5_CORRUPT; + } fts5BufferFree(&term); #endif + + fts5StructureRelease(pStruct); fts5BufferFree(&poslist); return fts5IndexReturn(p); } diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index e888abf21..4efc53219 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -3701,8 +3701,9 @@ static int fts5IntegrityMethod( " FTS5 table %s.%s: %s", zSchema, zTabname, sqlite3_errstr(rc)); } + }else if( (rc&0xff)==SQLITE_CORRUPT ){ + rc = SQLITE_OK; } - sqlite3Fts5IndexCloseReader(pTab->p.pIndex); pTab->p.pConfig->pzErrmsg = 0; diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index bcad9e724..184cb77b8 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -428,7 +428,7 @@ do_execsql_test 15.1 { } do_catchsql_test 15.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: checksum mismatch for table "t1"}} #------------------------------------------------------------------------- # diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test index 0abd8b86d..8788bc2ed 100644 --- a/ext/fts5/test/fts5corrupt.test +++ b/ext/fts5/test/fts5corrupt.test @@ -47,11 +47,10 @@ do_test 1.3 { DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } -} {1 {database disk image is malformed}} +} {1 {fts5: corruption found reading blob 137438953476 from table "t1"}} do_execsql_test 1.3b { PRAGMA integrity_check(t1); -} {{malformed inverted index for FTS5 table main.t1}} - +} {{fts5: corruption found reading blob 137438953476 from table "t1"}} do_test 1.4 { db_restore_and_reopen @@ -61,7 +60,7 @@ do_test 1.4 { rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } -} {1 {database disk image is malformed}} +} {1 {fts5: corruption found reading blob 137438953476 from table "t1"}} db_restore_and_reopen #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index 6b4d6d411..fd2a841c7 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -109,12 +109,12 @@ for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { do_catchsql_test 2.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check'); - } {1 {database disk image is malformed}} + } {/1.*fts5: corruption.*/} do_test 2.$i.3 { set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}] expr { - $res=="1 {database disk image is malformed}" + [string match {*fts5: corruption*} $res] || $res=="0 {$all}" } } 1 @@ -160,17 +160,17 @@ foreach {tn hdr} { close $fd set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}] - if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + if {[string match {*fts5: corruption*} $res]} {incr nCorrupt} set {} 1 } {1} if {($tn2 % 10)==0 && $existing != $hdr} { do_test 3.$tn.$tn2.2 { catchsql { INSERT INTO x3(x3) VALUES('integrity-check') } - } {1 {database disk image is malformed}} + } {/.*fts5: corruption.*/} do_execsql_test 3.$tn.$tn2.3 { PRAGMA integrity_check(x3); - } {{malformed inverted index for FTS5 table main.x3}} + } {/.*fts5: corruption.*/} } execsql ROLLBACK @@ -209,7 +209,7 @@ foreach {tn nCut} { set res [catchsql { SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC }] - if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + if {[string match {*fts5: corruption*} $res]} {incr nCorrupt} set {} 1 } {1} diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index 437c4842c..eab4c3c91 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -102,7 +102,7 @@ proc do_3_test {tn} { list [ catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg ] $msg - } {1 {database disk image is malformed}} + } {/.*fts5: corruption.*/} catch { db eval ROLLBACK } } } @@ -273,7 +273,7 @@ do_execsql_test 6.1.1 { } do_catchsql_test 6.1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------- reset_db @@ -289,7 +289,7 @@ do_execsql_test 6.2.1 { } do_catchsql_test 6.2.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------- reset_db @@ -308,7 +308,7 @@ do_execsql_test 6.3.1 { } do_catchsql_test 6.3.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.3 { ROLLBACK; BEGIN; @@ -319,7 +319,7 @@ do_execsql_test 6.3.3 { } do_catchsql_test 6.3.3 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.4 { ROLLBACK; BEGIN; @@ -330,7 +330,7 @@ do_execsql_test 6.3.4 { } do_catchsql_test 6.3.5 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.6 { ROLLBACK; BEGIN; @@ -341,7 +341,7 @@ do_execsql_test 6.3.6 { } do_catchsql_test 6.3.5 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------------------------------------------------------------------------ @@ -374,7 +374,7 @@ do_test 7.1 { db eval BEGIN db eval {DELETE FROM t5_data WHERE rowid = $i} set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ] - if {$r != "1 {database disk image is malformed}"} { error $r } + if {![string match {*fts5: corruption*} $r]} { error $r } db eval ROLLBACK } } {} @@ -399,7 +399,7 @@ do_test 9.1.1 { } {} do_catchsql_test 9.1.2 { SELECT * FROM t1('one AND two'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_test 9.2.1 { set blob "12345678" ;# cookie @@ -411,7 +411,7 @@ do_test 9.2.1 { } {} do_catchsql_test 9.2.2 { SELECT * FROM t1('one AND two'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -497,7 +497,7 @@ do_test 10.0 { } {} do_catchsql_test 10.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -678,13 +678,13 @@ do_test 12.0 { | end c2.db }]} {} -do_catchsql_test 11.1 { +do_catchsql_test 12.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} -do_catchsql_test 11.2 { +do_catchsql_test 12.2 { INSERT INTO t1(t1, rank) VALUES('merge', 500); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -870,7 +870,7 @@ do_test 14.0 { do_catchsql_test 14.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #--------------------------------------------------------------------------- # @@ -1040,7 +1040,7 @@ do_test 16.0 { do_catchsql_test 16.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1126,7 +1126,7 @@ do_test 17.0 { do_catchsql_test 17.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1435,7 +1435,7 @@ do_test 18.0 { do_catchsql_test 18.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1546,7 +1546,7 @@ do_test 19.0 { do_catchsql_test 19.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1630,7 +1630,7 @@ do_test 20.0 { do_catchsql_test 20.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -1764,7 +1764,7 @@ do_test 21.0 { do_catchsql_test 21.1 { DELETE FROM t1 WHERE t1 MATCH 'ab*ndon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -2100,7 +2100,7 @@ do_test 22.0 { do_catchsql_test 22.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -2211,7 +2211,7 @@ do_test 23.0 { do_catchsql_test 23.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -2429,7 +2429,7 @@ do_test 24.0 { do_catchsql_test 24.1 { UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thread*'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 24.2 { INSERT INTO t1(t1) VALUES('integrity-check'); @@ -2518,7 +2518,7 @@ do_test 25.0 { do_catchsql_test 25.1 { INSERT INTO t1(t1) VALUES('rebuild'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_execsql_test 25.2 { PRAGMA page_size=512; @@ -3011,7 +3011,7 @@ do_test 27.0 { do_catchsql_test 27.1 { DELETE FROM t1 WHERE a MATCH 'fts*'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -3700,7 +3700,7 @@ do_catchsql_test 32.1 { highlight(t1, 2, '[', ']') FROM t1('g + h') WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 32.2 { SELECT * FROM t3; @@ -5351,7 +5351,7 @@ do_execsql_test 41.0 { do_catchsql_test 41.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 41.2 { INSERT INTO t1(t1) VALUES('integrity-check'); @@ -5573,7 +5573,7 @@ do_test 42.0 { do_catchsql_test 42.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: checksum mismatch for table "t1"}} #------------------------------------------------------------------------- reset_db @@ -5813,7 +5813,7 @@ do_execsql_test 44.1 { do_catchsql_test 44.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 44.3 { SELECT snippet(t1, -1, '.', '..', '', 2 ) FROM t1('g h') ORDER BY rank; @@ -6644,7 +6644,7 @@ do_test 48.0 { do_catchsql_test 48.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: corruption on page 1, segment 1, table "t1"}} #-------------------------------------------------------------------------- reset_db @@ -6917,7 +6917,6 @@ REPLACE INTO t1_data VALUES(1,X'2eb1182424'); REPLACE INTO t1_data VALUES(10,X'000000000102080002010101020107'); INSERT INTO t1_data VALUES(137438953473,X'0000032b0230300102060102060102061f0203010203010203010832303136303630390102070102070102070101340102050102050102050101350102040102040102040207303030303030301c023d010204010204010662696e6172790306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020108636f6d70696c657201020201020201020201066462737461740702030102030102030204656275670402020102020102020107656e61626c6507020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020202087874656e73696f6e1f02040102040102040104667473340a02030102030102030401350d020301020301020301036763630102030102030102030206656f706f6c7910020301020301020301056a736f6e3113020301020301020301046c6f61641f020301020301020301036d61781c02020102020102020205656d6f72791c020301020301020304047379733516020301020301020301066e6f6361736502060102020306010202030601020213060102020306010202030601020203060102020306010202030601020203060102020306010202030601020201046f6d69741f0202010202010202010572747265651902030102030102030402696d010601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202010a7468726561647361666522020201020201020201047674616207020401020401020401017801060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102ad060101020106010102010601010201060101020106010101010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020415130c0c124413110f47130efc0e11100f0e100f440f1040150f'); INSERT INTO t1_data VALUES(274877906945,X'00000e96023030011a042319320d3b123d812b5a31120110446e581b66814a05010a4537814274010e8102815c810f3d0104846d01081581204401103741043c59416b44010a404655265301103f73811a11114213010a821235820f020135030484320201360104816a020162020484550302390301710a04824a020166030483690201670704837d0201690404822602016a0504825c02026b620504817502016f0904810d0303e79c88060482760201700a04826302017204048155020373c2be050481130201770204846202027962050482710202c2ba010482140203e58496070483330204e8b2b879010483710101310110545c0c814b0e3a6501082c815d5b011a2a0e2f0d765c3d686014061d0d0112810733112c2e82141101048313010e5c6f632e813e42010c811882370548010e19158146822f1f01104d364a708146135a010a237b0a55210201610904841703027678090481270201620304810002026374060484660202657a0704827602016601048351090483540301660704814b02016b03025f0304c582caba0204816602016c01025f0302cebc0904843e02016e0804821802016f0a04817503016f070483100201720304822c020484380201740404842e0102460201770104812f0204836c02027a6f040483110202cebc02048267040161020484650205d5bd62cebc0604845b0204f2a580880204842a0206f38184a179670502750204f696a3aa0a04814601013201063330390110812281378114600d010c03716c5e822d010e81226b542a814d010a72740f83000108813a1e0b010c5681046f812c010c07814a777328011664244219531b1a2f811e4a010c4d81557c7f1b0201300704810702013307048230010484050202367807048175020239710804832502016204026b0204814d020363306108048262020265650504817602026667070483150201690704832f0301360a04814d02016c08024702016d0304843e0303cfb2630204814002016e0804837503016203048416030370c2be0304821b02016f0604834b0201700504816b030175070210020273660604841c020676c2bac2b2640604830a02017704027d02017808048141010482700201790504811d0202c2ba0502470206ca8d73ecbab9010483340204f09e9ab504048367010133010c3e04814f82250114812e814b2e0411811305010c811337811e6e010c82085e2b0e5d010c61812054811c01148122451b0781050c813d010c17823762643e011e080c1720814a10364306143b0d33260112810f0c2a810c816b13010a8163810e470201370404811102056176c2aa36050481530202646e0904846202026a730404827402016b0a020c02036f616b0404830a0201750504820d02017605025e0201770a04820702027a73050482460202c2ba0604824a020483330203cba434040483200203cebc790304847e01013401124181442c1d091f81580108601d8336011081320a2b8125820001123b0b81158116811f070110078112817a817308010e6682410d810e2601122d0d6413378147351e01105081021d3525812d01128246510a622204054101105c1b620e81302b05020130020483480104822702013102027e020132030483270201350304844802023770030207020261710604823f0201640802570204830a020265770304831f020168070483210201690204825b02016f020481280704835402057037e18b8d0904810802017304048439030172020481440303c2bd6b0a02630201760202490804815e0201770304816e050483550201780704816902017a070483280207c2b2f093aabc780a0482240301b303024e0301bc0604837a0202cb800204834a0202cebc04048201040484410203d3ad770a04814f0202d5a508048371010135011630817e0f81040d2c041552010c813d3b7e8115010c40692182693a01121f810d810d0a32814701101d1d1f642281742e01068229240110811231810a387f4c01100f50810f8165810d0114811f26443152593c104a010e641e1a3357820e020132030481540201340204815402023778040207020163060481020204815b020164020483010201670a0481540304f0948f870904811a0201690704835502016d0604832203027b01022803017a0a025102016e020484260404816002026f690502680301720104834e02017208024f02021d020475e69c8e0504814c0202767602025f0302de870a04837b020178090483200104835003026a72090484400201790504837204022302017a0104836b0202c2aa0a0481070303bcc2b3090484370203c7866403027501013601087c158303011212814305813e7b0e090118141a1c49713a211e0c74630f010a59826d8113011203328166037781561a01101d7f1d2a1f822533010e820e070f7b40160110811f40292c813226010a2d20824a32010a81418158670201300304840a0201660404817d02016d0804826902016e080626817f0104820b02016f0404825a02017003023b020272750804840c0301760a04844d02017403025b020175080484200202766e060482360303c2ba6c030482220202c2aa0204810e0301b20204835703048421040484240301bc0802410202d2a1010482630204e1b18f3704048354010137010c08816337812f01101382211532424d39010881248123010e7724810267815f011081236029813e273301101b7b29812a5b813b01128150810324814b220b01060c8417010e165b6c81708117010a1782346f6c0201380804816803016b0604840102013908023f0201610204816c0201630302760201640304832604023e0204833902016502048203020266770804821a0201670a04830002026964080484500304f29e9eb70802250201700204811b0201710904832d0201730304826b05048403020174010481000205776a62c2b2050482630201790202260206c2b2eaaeb464050485020301b3080482480303bac7af0a0484550203c695650804822a0202ceb90a0481170202dbae05023f010138011819814a2703390a61090c6912011a21181304812523811b5f164e050114110f35128123423f810c010c817573817c03010e7182590c812b0401142503597e6e0e2f3a3759011252813a811a2b75091a010882162a31010e17450a81048279010858658208020231670a048205020361c2b906023b02016301048236020164050482520201650904833d0201670904811b0201690604825002026a7a0604837c02016e0204832002017002026d0302c789010481020201720504835e0201740604810002017502020a0702630302676306025f0303e4a0a70102640203786a75010484440201790104841402047ac2be72070481340207c2ba3766673576090482790301bd07048142010481600202c7b309027a0401740604823b0202d2bf0304830f0204e989a6300a02600204f4bd91b60702120101390106518369010e19254641823711010c258267288121010e817c810d2b17250110810a578133812f4c0110415681067b288121010e0881208119347101140b8131543c8100343d1101088203813e01100d742e3230820f3802013006048107020134080481440202356501021a02013808048147020162020482230201630304833c030162050483390206656cf093b5bd070484140202677303048502020769e3ad9669c2b90304847402016b0804836402016c0404841f02016d010481250904825202016e06025402016f0704842a0201700a04834a0201720a0483530201750304822e020676f097b18374030482140201780604833d0202c2b9010481550301be070483720204826f0202ca80060481630202d5a80504833f0101610114551047810e130a78660c011a364611206c0b13080705733d5501240f08070c090b0c20813d1471042e4351131e011204412d814f0913104201263036110d060b1f811a301b0f4e1a29092f181c012808071e221a2a81075b320503065a0f140c1a0a26011c07231d0e6f3715063b760c6b091501121111303e3a71566d6d010e0867814d816a0c01181e18240d41724d221b3f384b0201300204830b060483080302c2b30204837f0201310a026e020134050481560301730902690201360406827c0f0201370204825d06020a0201390904815b0304f3bfb2a70a04822c0201610404810e020262660604841a03017608025b02016309088112825f020164090482310201650a0484480302396f01025d02026774010482090302df9b06048321020168070661813303016f03048248020169030483610504814001048401010483460301300704824203016a0204824402016a0504813303021d03017209048412030277380804824502016c0104814c02036ec2ba0804835702016f0204811d030176020238020270360a04840d02017107048201030469ca99690602350201720304812d0104845e02017304022c02017409022b020175050484140302caaf0402410201760404831f020177010484180704845203026f6d04024b02017a030482660202c2aa0204810a0301b30a04817b0302bd76020483780302be6a0302440202c58207025e0202c69901027a0302ad77010483200206c993f099b183070484270203caa1660204841a0204f29788ac0804831d0204f4ba9f950504843c010162011c0e33810216341c2413042130780501184d373e53131f2f052907423e010c830e3781390e011c1320461f81041b811b041e15243d011e241b10816c310b130c3133033b0741011a11816d3139100c13140b395848011c580e411a06304306810a3138330d011a441707092c70140c1643813920010c73653581374f010c826c81210f0402013803024f07048172020161020481650204810c020162030483470301370404813602016307048379020483280201640304815501048176010481060201650104812a0104841f0201660504821f0201670302700201680804846403016b0a04831d0301700904845502016908025c02016a010483560904827602026b6306023402016c0404832602016d040484410204825702016e0504831603027831020482160302c2be0504827d02026f6a05048121020171030232020483220304845402023a0201720204845c0304846e03013607048224020174060484480201780704844303016f0604814f045807070a0707070707080709070709070808090a5c07080708080b07060a06080707070b0a0b0808070b0a0b0a55070b08080a0908080707060709070709070706080c060b07070c0a66070b08080609070607080c0909660b06070707080a0807070b0b0707080a0b07070d0607080c0908630707070b07070a070d060b0707090a07080b080a070809085f0707070c0706080706070809080f06080a5807070607060e070807080907070b070b060c0709090807690808070707070708070608070709070809070a0d0b07070809095b070707070707070c080d07070b06070707070c07080b0808811a0b08060706080a070a07080609070707080808071307070a0708070907060807090b06060707070b0707080708070707080c090a0a81080a0b07070b0f0b0706070707060b07070b07080808110b070707'); -INSERT INTO t1_data VALUES(274877906946,X'00000e880330627a020482240202c2aa0a04833f0301b30704844b0302b9650704824f0301ba0204845f0202c9820a0483640202d194060482300203e19cbd0904844b0203e691b4050483510205e78dadde9b0104821201016301142a6c033b8151085c094601140b813d49313f81110e1c011681163611221527257f5d38011c150f22811a0a3c12350631238117011c3e26420b402c1d81080c40150b2f01181c3143382640273d60132e070118663b1d162a1b0e2e8111393e0110821117310e52811c01141a2f49810181391f2b130112323c0305812a6f2e390201320204842702023334020481340201350202610201360304844603023362040484470203376a360a04826b02013808026203016f0704830d02013902025502016106088170827a0301320a04820403016c0404831502016204048327050484030201630a04814302016401048349020484760302430301640204845d02016504048249020367c2bd07020a0302cebc0902150201680a02500201690204846102016a06024c0301770504842b02016b0104830e0704811803023370040483580301710404845f02016c0504844f0204820d0204837f0302c2aa0104833702016d0104844c02016e0804834f03026c6a07025702046fecbd9a01023a020270330204830a0301740304837c020271350204811e0201720706833b310206736ef09289b70104832b020174070483290301320204827c02017608022802017806025b0302c2b30904835202027978080483040303c9b56f0904846a0202c2aa04048127050482120301b30504813a0301b901024204016a0704840c0202c5820704823c0202c999010482470202cebc0602400203db91670602730202dca7050482760203e1a3950304817e0203e786a702048273010164011a0612105b292b817c1211080d5a01147c1d420b35451c36811a011e0e168117081c0c2e051d474055192d011e02050a1c81180420250f815f300f21011c02316a37143321443a10042d54230112761428810e4750054101101805072b8215294e0116680f0f5381445a3e0b070901224e4a41210c361c281b101c43051325130f01185a1e19108106300f2e3f4538020130060481370202327305022002013405048168020335377a03021802013704027b02013804048260020161030882118101030263650904814d0201620604822502016303048419030135090483240201640602280301380404817e0201650404823c0304f097ac9f010482680201660a0267030566e2b6936f0104821c0201680704813302016b0604832002016c01026a0301610204843202016d0804845e03026d6d03023c020270730a04817f0304c2be797a0804832e020271710504835f02017203021b0201740204825a02017706027202017805048451010683572e03016e0a04814c0201790304811702017a080484450302c2b90204837b0202c2aa0604825b0303b273630104841c0301b9050485040301bc0a027b0303bd37680502670202c98b0204826b0203cfa1740504823a0203d199610202350203e3bf87040483570204f1baaba90504817301016501120b8104392d0d20180f011645213f292e4d0d082f8165011e0b400c07341b2329307f193338173a012055292409050c560a272a0f4403245718011a1c3a183f1c43264c3126060829012081208102043a044d0621650b180e150e011a066c030e513d7d265e1313130c0118171953040457347b114d191901261b1c060c26090d6f0d332a1519096e03101d1d012207342f1f2c7e2517251d0f310d2a17081e02013005020a02013308048247020135030483660201380704841b030132030482180201610102600604825f0304c2bdc2ba02023a02016209021f0201630604813002056663cebc610604841e0201670504816a0104842703037177310604833f03027872080482350201680604825e02016a040483320104840d02016b0304813f02026c6408020602016d070481590104837d03016c08020d02016e060484630301780404815c02016f0104826b0804825e02017008027b0302c2aa0504847c0201710404836d0201730402510302cebc08048338020375c7bf05048344020177010482660304822303026479070481630204786ec2ba0204814f0202796a0a04834f02017a01048407030484660604810603026561020483180204c2b278390504813d0301b304024f04026536010483110302b9330604813e0301be0304840b040484560202c4a702022d0206c6a5f7ada9990402350202c79f090481180202caa60502140204cebcc2bd080483320202db900a0481250205f4b5aa9079040484360204f7b985bd0204835e01016601128101285c096981190e01121f813f0d431a8135530114698102813228492f190a011260161881328101812601188155780d813257050c0b04060114161681340772811b5e25011c4505810e13290b253a0c0c0a1a4b011a3714133e1235812b136b062c0b011a6b591356810c3c240906250b1001148127810d413e0e81090d020231680504822c0302c2bc0204830c0201360104840c0303656a740902110302d1950504824f0201370902130201390a0482530303e0a9ae0904844c0201610804810d0201620a04810c03023039030481330302356902048268020163060483470201650504822d02016706048200030483560302713509025002026a79010484410804825902016c0504822002016e07020d0304843a02016f0204842d0201710304837604048361030482430201720904840f020173020482520804810b02017406068425320201760202420201780804845d02037979650704814802017a080481110301780804812c0202c2bd070484600202cebc0508813082410102770401610202250202ddb40302310205f19e9a937a030482410101670118365558195a0a062d0581260a011881068143330844041f0a1851011a2025141e1081204f550e077521011a193b1f58351912265681220821011812070528472f4e2f407f204a01124d1e1f811b810d7b4d01180f1d3481034a35580a12811f011c2303340d1470150778070c812331011620411939703c032915143f01104281116a3d323c6a0201350604842902013703023f0201390a04816e0201610604826402016202023f0201630a04843302016409048258020165020482620404814702016602061b833f0201670404827d03026369020227020169040483490301670604847602016a0604845d0404840902016c04024601088150822b010482350204830a0301690304844902016d0202710704820002016f03048509020482380304836603026e74010483580201700504817a020171020481080204826a0201720304837b0202410302160302c2b3080484550201730904816e020174040248010484280304834e020275730404836e020176060484720104815c02017709026a020178060275040483790201790504821501026c030170080483770304cfb269710704815102017a0204813303016f090483160202c2aa050481400301ba0804810b0401630804830d0301be0604844d0202c8a30a02110203cba0640204816a0202cebf060482420204e487856e030481080204ec97bd6d080484080205f09eb3a0770502260208f687999931ed878703048424010168011a12460e090c036e151b812e065501161708411982151738471f35011a2d1c0678340c1f04425c21200c010c2a087f255d4a011a0d0b6c33814a212c3a0a401b1e011c501f2381010a0481201c0c6012280118150b5228520e0a036c1c8123011a15810a060408030a81563f381601185b1b06212a143f332a60160e011a221b1e62411d2048090e0b0f5502033072350804826a020131020482530201320304823b0201330104814502033677380204813102026174020483540304dbbf6f620404835d0201620504846a0104831803017103048323020164060483740201660a02410201690102130104821402016a0504823f02016c0404832e02016d09022602026e640a04822702016f070482000301750402670201710304813106020d0202726304048220020173060484530201750504831e020483400302c2bd0704843a020177020483470203786371030483740201790904810002017a0302300202c2b9080483280301bc010481700303bd33720304825e0205c99973c2bd040483160203d5a6330a04842b0204e7b3b3300904813c0205f099a68f72090209010169011c21101d4b2d0e0e066b4253074c140118070a0910447556030833541d01163733816837402b3909122501183c5b1139102e2d430c662334011e27050f21621230323503332b6a0332011e1e07031843202e6e3c2850094d410c01163955220b16812d24521212011681250b0a3505460481176f011a2c09266b162968051c0a1481170116022e1e820c352037263a070201310104825d0602110303696869020484070201320204826e0201610304832203016f040232020162050642843f03048336020484540301370804833c03026c6105027a0203636165090483120201640502770304833e03017107022f0301780702470201650a04811e020167080238020168040481160102230404826b030170040483000201690304836f0302766c0304811402016b0504812e02026c6108027702016d0308827d81530604837302026e790904842602016f06048208020170060481680302320201710204812902017307048255020274320104822a02017506026803016e0a04821303017207025f0302c2b90504834a020177020483130201780604836b0402210301320604847302017a010483130202c2aa0804823c0301b9030482600301ba0104845304016c0504837e0202c3b8080484600204cf9d6379020483660202d3860704812e0203e3a4be0402560203e58784010481210204f09e95ac0102580204f5aea5890a023301016a01123428131a1f6c81445601141e227c1a7b5f1918810301182318812e17455605460d811c011a28820221311a6e12093f050a0c0120082c0b0f1362074457460c3b070d5132011c2143052a20133d160a358117591f01103136813b136e6247011c100e4c28060d16815a320a3e11070124462c03582e262d45110804113326040808070807080809090b81050708060708090607060907070b070e07070807060706070b08070f0807070709080708080c070706060808090c07060708080708080909811307070708060709080707070607070a060b070706070707080a080607060c070707080809070608080908090a812406070707070a0906070b0b0908070b07070b0607070b0608070608090b080a080f080a0608080b070b08070a080b0a810408080708080607090707080807070b070c070a070f070b080607090707080d06070b810b070607070607070b08070707070b14070a0f08070b0d08070e080b060a0a070a0707080707070709080a0a0a0e810f0907070709080a0b0707060a0707060807060a08070b08070907060807090b090a0a81140a09070706100707090a060607060e07070807070d08070a07070806070608070a070708070707080a0808090909'); INSERT INTO t1_data VALUES(274877906947,X'00080e7f073c23110a1a18392f66090524183704276d6703306a320404824e030164080483520305c2bd7ac2bc0604815a0201360704833202016106021f020482400201630304822a0708817e8204030173040483500201640404824803016d0804824a03017709023002016606048367020268680a04815802016902088339811804027f0302656e0704834e0303d5a5370604816702036a3366090484470303c2ba660904826e02016b0904837c02016c07021403026c610604835802016d0204816802016e0104831202016f0104822f020270720602060201710704822202017206048174020273690204824602017409020c020175090482140201760a0482720301660404824403016a090484290201790404845703025d0203c2bc33040484620301bd0304824c020484540202c78607022403019a010482380202ca87070484390202d39d030485050203e184940404831b0203e6a881060483480203e8b18c0a04816d0203ee8d850104814801016b0110467257393c81272c011a053e815d3b190517064524521f011c3823590a8115372004313b1f3216011a5a20780b102d0804426916112c011a182f810781082d12137026161501221a180516811611051c131207811515173501180320112581062e05621c1407011c2d0e0617811522062208065a21520114582841621e6c203f1e2001161647411a272533815b1c2602013009048309020232630104835a0301720104817f0201330604836f0302ddb5080482560202347a07048102020135020483460104827b02043678ca800a04835f0201370404814b0104846002016103048246010482220301700204833f0201620404824d060481150201650304824f02016606088110834c0201670604821d0303c2be66010481790201680404843b030176050482270201690a04830e02016e0904844202016f040481010301630304822f020270640204822f03016e0704845802027177090482710206736ec2b2796a0104832e0306dab1d485377004048304020174050481700201750a0212020378627604048164030173080483190201790704833d0204823a02017a0506820e67030178070484530202c2b2060481500104823f020483030301b3020484310301bc04027e0402caaf0a026a0301be040482590204842e0202540202c79f0804824d0202cfa30804815a0204f29a92970204823301016c01140f63351a0a653b650d22011c09117a3e1538123537046a15043101141310082f49052f772b0c011c11121781583c2a5010133228241301287f3e0a2b1244080503060a100f413b4f0d070e2a01103e4e1f04814e7b1601183d0404052877111f230f811d01123a100f053e5c076910011a031732102381243d1b1727507301180e5d273e810803812e0f192a02013301048271070481330204821d020134080263020135060481280201610104830a0201640604826d020165060483050201670204841c0504841c0304841b0201680a04845602016a0104811c01024f030481080204813102016b0204837008024502016d0404836c04068207780301670704842302016f0404821203016d040484490301720404837e0201700104821d03048407030165050483050201720a04811602017307023502017407020503016f080484240302c2b90504821b020175090484090201770a088119822503026d6905048300020178030482680604812a0201790104830c0204833a0303d9a06806022002017a060482600203c2aa33030481560301b904020a0301bd0504820f0202d0b90904817c0202d3820202200202daa9080482030203de966e02024b0202df9d080484350204f098b0a20604845e01016d01220304456608322258060a031d4c38340f090112310c070e4238626e6601124a318109030513812f0118240d561e533742188113101b01160b24444b224d44814d4806011c05774e483410330d23541b28090401141f29062581131e221b6d011e81053a037a03320b0e4c24360d2310011a0e321d3c141825111d54637a1c0114093d3c2e58571a35293a0201350104840a03017701048330020136060217020138030483370201610a0482650201620504815e0201630704827701048201020164080483690201660804846703016904048113020167070483080201680504837d05022c0302cdb10a04815f0201690104833a0404824302026a360a04823b02016b0a04813502016d0504831a0204833803021a02026e360404825e02016f080484140201720304844b0404816603056ff09d899b0304823f020275390204816e0301780a04824202017604088308812703027902027770050482040201790104827e040482750204812902017a060483030304c2bdc2b30104836f0203c2aa62040484040301b903021c0302bd6b090484300301be0704814d0202c99402025603049a65656b090484020202ca92090482060203d19a730504844a0203d49f690804836e0202dfa8020482710204f09180860704822901016e011a0c0b8104243647521f43231f36011a2e1b33432c3d0b414905054d17011010573a6c0a816c1801160e063582340a5239050b06011a4481063d1b67250f2044200839012044591d1857291214135814101a1b361d011225067e8147111a4a4301166b13362e17195f3812186f01141c465b032b290406373301182a152a2281300f8107054e3f02023274080481770305c2bacf8168020481450201340604832f0201350704842e02013605020e0201380404841d0201610404810d020483750201620304812b020484230301610804834503026c6a0304816d0201630102380305613577337405048359020165040482720201660904826202066736f094b0af0a0482250201680104811f02016b0304847202016c0404822403016f0904822c0302c2b301025002016d0504817b01023f02016e020483090802040303e7bda10804832d02036f6b740404811402017005048419020484220202716506026303026b760904830a020172080482430304706c73620504825f02017308048413020174080481070201760104827f0204836e020477e7b89a0104840e02017a030483700206c2aa35657065050482740301b30804842b04046cc2be78090481040301b903020d0301bc010484260904813f0203c7a5620302330203c99f36050481010301a30704815b0202ca8b090483250202cdbb0604820a0202cebc0102170401380304842b0207eca2a6f29c87950904824001016f01221d17052b58101241060e3a201f1021633a0114816919811c142443100801280426080e2620042a812c531a490e121707131710011273432e493347811a340112195f671f46721c325e0118380c052b812822478107600b0116021c21821b2019263433040126021b05351b2a286b05181f071b5628111a330a012014533e073d0c0e5469141d1e2734050901220318051b44412803632e0642370e0a3a2b020131070481770201320a04812f0202346e04022b020136030483590304f09a81b60404834702016105048210020162030205030167010268020163010481540604820202048300020264310902420201650804834b02016703048247030365c6b602048205030573f098b890030481450201690204832802016a0a04826703016c0104825e02016b0604815e03016c03048334030677c2aa74c2bc09023d02016c0304823903027777060484540303df866c0104815b02016d0204811f0303796f7704020302016e0204814d07024a02016f0902680201700604840a0104831a0204835a020172070484440201730a023703026b660604830e030278790304815b0201750904822402017704025b020178030482350307f4b2a3896a343407026b0201790902720302633409020402017a020484590302dea004025f0202c2b20204816b050481200202de900402160204ee85a5770204822c0101700114143a0d391a60812d4e09011a2f313104201c372c3a3411321b011a268140144226334145050d1c4d01164e081f20671f088107237901186b123c1f6d07261e2b732e210116511116342a3d32376e083001106882257a0a17141101163039192b0c05812d735f3b01262a3e0841030b17181411051e0a18530e272b6d01182f322b260e24581d5381050f02013104048353020132050482370301690204843e02013302020d0201340104841f0201350a048139020137050482770201380204833a02016105025a0504832f0201620304836d020163050484100304832003016c030481290201640304837e020482490304822b010482290201660a04827002046964dc960204833102016c0704814502016d010482000201700104817e02037176760904821f020473eb91a708048152020174090482770201750404831e01063a825d03017a0904826a020276730a0254020177080260030277630104815c020178070481220202020301720804841202017a0204834c0202c2aa040484010301ba080482580202c6a3020481320203cdbf690502790202ce90070483140301bc030481470205d1a371cebc060481590203d2976a0404830c0203dfba6e0604814b01017101163732393b8120422f054b0e010e030b211d815d1c01165641757c080d81311d090e0112816581542d2313054301224e07121706516606080e39102d231c4b39010a2d81402d5e011a132527428114080d6e1111721c011a814a1a341538251023100d1c4c011e22182622623712411e38162a182d3b01142b67611981470f1f1f250201310a04824e0202336207048217020238730204815a0303cf886d06025a02033962620404833803016f0a04814e020161060483140302726d050483450301790904810c02036376690504811c02016403024d020165010483280802550301650904827f0304ebb8b561070482340201670302670301660804810e020168040482340201690704844d0302616404020f02016a030481060301700704827802016b070481240104814b0302c2bd0504816102016c0604837f03017a0404837902036dc2b90804810002016e0904821602016f0304812c03016401024b02017103048233060483600303e5848e04023a020172040481050305f3978aa06c070481151708070b070a0d0707070607080c080909090706080707070707070806070707070a090b070708080909090981140708070708080b0a0b0b070b070907090707070707070807080c0c070609070b0807100706070e08080a81110f06070707070f07120a0c070707070b0707060607080709080b0b080709060708070808080a810f0707060707070b070707070a080b08070e08070b0b08070c080f070a09060807070a080909080a810b080b070706070b0b0708060b07070c07070707070a0a09090b0708070a07070b0a070c070a060b080907080807070d8123070707070a0706060f070707090b07070707070b07080907080a060f070608080706070c060707070c070a810f07070706070707070a070b0713070a070707090a070c070706080a07070807080808070b0909810607080808090707080709060a070a060707070707070b080707090707060b0807'); INSERT INTO t1_data VALUES(274877906948,X'00000e8a0330717304048359030134050481100203756371070482190201760704817b0301770804821d0201770204844d0201780204836c0404826103017504026a0202c2aa020482420301b305022c040267390602570301b9080481540301bc0102290206c99cf6b5aa80080481430202cebc02026e02048120010172011a1c2f15158108048125463f251d010811811539011412423105812181171549011847284a30234e5b33042632120118351e8113817d0f2b220d111901264f104a211004061d0a2a0b35121a0a2118341f011c81160a1b030d2a0610243e445f0c011c6f0c1e3b1768141e322717500b140110537f810169811625011a492847203e210f532c16480627020135020481780302caae0104811a020261330904846c0201620804812d0201630404814a0201650704837503017301048276020168050213020169010484540604842b0302796f0504833302016a0304831b02016b0604701302016d0604815302016e070483630204815202027071020481520201710104835b02037266700704843002027362040481490201740904817d020175020481040304f59e9c9407048218020176060484060204776dc2aa020483070201780504812503016601048159020279790704840502027a730904826c030178030482740202c2aa04023d010483540301bd06026b0203cab877010483290202cdbf060482410202cebc04025c0401690204827f04016c0904840e0202cf880104835d0203dfbe6c01025b0204f0aeb7b2030481680205f1a7b5bb390504826a010173011a22810d12415003071f81181839011a3220221511546d810012052b57011a0c4274300d154e81111f041e10011e293f4213051b2276560817312811170120092136122418370e4e782b3912080f3201262b3b340f222b0c09142a0822116a135c1c130c0114320c4e385a0d0415075f01163543340f06362381133c0c012224180981742048191d110e180e180d310f011a20632450281f043027114b034e0203316a61030482160201360104826e0201380a0484570201610104844d0504814a0201620904820f020164010481630304810403027761040258020165080481550302c9b70204827a0204677367690102660201680104831b020169020483760301720a04814502016a04020c02016b04023405023c030269630504840502016c08048463020170030482670202716d080482000201720304835c03016507024d020174030482520504821503016505048207020175040208030137060481710201760a04833d0201780404832302027a6f080481200202c2ba0204812c0301bd0404821e0205c3a66865730702540202c7890804816f0205cebcc7af730504815c0202d2930202540202db89020481160203e8b8a00304825d0205f0958db331070481620305989b8569780a04813d0204f69299a5020210010174011a7b3829100a4e511f1a281c17140114812626032c372634234c01140a520e815a810815200501123e4f3531042d57615b011a041f3e64070f1f1913274a20770114811d0f5d743e0634161c01162c2782130c1b810520280d01164a513110480b402b810d13011e522d08042c1146137012201e810512011a290903182c05301e5d811944290201390a04836d0301610104836e02016208027803036cc2bc07048261020263640a0481480301730704813602016408022b020165050484310301720702260301780402500203666d73070484470201670104825b02016a040481590304836702016c0304835601025e02016d060484110301340204813702016f04021802017006068336280201710304813507025902017201026508022903016d0604812f0201740304827e03016a010482440201750404834b02017604021a010484150504836a02057773c2b2380602520201780204823e0302cebc080484040303d2956403048171020179020484240204813c0202c2b303048307020484410301be0204816f030484250203c798680104843d03019c030482570204e19ea86a020482350302a0950a0482280204e5a4bc780304810e0101750126090a35030a03220a1731630f31252f0c4b1e31011e39200e3715282a03103b56090f6b1501121d4916246e6d460d6501162609380406361e816d203f011a22166008124f58202e182025150114390f3a25713f0e3f715c011a5a11191123466025710c313312011e3c191326811c1444055f1f5109051201143b106f1181000d068155012043381381020d81080d0603171824260a0201300404844a02013207024203026b390204833803066eeebabb35660604842102013308020302023f02013503048243020436716b66040481440201380a04843d020261690704836002016203048209030484670201630a04841d020264790104822b0201650502340302c2bd060484300201670202620201680704810d020169010484430104843402026b67040481540306eea3ad77c2aa0a04836702016c020258040482270104830a02016d0404824102036e716a0604843e02016f0104832a04020c0204836803036530650a04817402017005025b0301630604843d020171070275020273720404827002017504048133010484120301370102270201760104814f03026203016d0904844802017701048375080481220301660202330301700a04827f020378c2b2030484710202796806023c0301730704813d02017a0404815f0104817202048407010483390202c2b204026802048254040266640204831e0301b3090482230302b963060482010401750a0483290306ba35f2999dac09020f0203ca926e090483350203cdb4780302350202cebc0a020f0204d7a7696d010483100206f097bc996d71040481480101760114185b2258291610821c0e01160272173107154f5b813722011a81020c200e1826250d39811f07011a7911152a2a45131504422c81070120050d3f5b23342e3e4139032a3813042d0116592d1c15630c0c0a814649011c1a362f5c4a35511f0804033e372b01102981262a352e8205010e0b4b6282388106011e26810a2d125f361a12170d1721311e0201300204832b0201320104811b050483790201350204832202016404024c0202657a090483710202666307022703016c07048362030277750604842f0201670304843d0104844c0203686f7a07027202016a0102430204847502046ce0a2b20704810302016d0204827b02026e330804826c02016f0104835502017102027606026c020172060483490202736505048371020174030484130302387602025d02017502048345020376346b0904825c0201770904814b0303c2bd720804812b020178040483600201790804816402017a08026e0202c2b90704811c0301bd0504821e0204cdbcc7a108021a0202cfb8080481490204f09f96a50204842101017701180207232d37812d0c045c4a0a011a06163b3408171c52213a26592201206d08581605811a171e0c0a1347104914011282181324082b73320f01122f6e811d2c3d410a44011e551414206a092f133f333d150a3e0f011e235b170e37060627471b13373b3e27011a0e1e816b270c10102d53381045011a060e1e254d044932651234691e011a158138300a04810c0a8121071802013003027707048214020131080481370201320a04835502013304048271020538ceb369650802310201390a0481440202616c0502570302c7a104026c0201620804811c02016301048168020241020164040481560104820b020165080484460102400202686c060232020269670604827302016a0108810c826e0704824e02026b6c0604816902016c0304831c02016d0304811903016407022b02016e01022e0604845a030237780702040301710804815102016f0104842903017a0704826902017003048445020482080202713002024e0201720804822003016202022302017303048111010482790204812d03067479c39f66700a0210020174020229020175090484430301690604820402017606020202017a01021b0104843a0202c2b90604842b0301bd030484570202c69b0504815d0202c8a30a02240202ceb8090484690301bc0404832f0202df85020481230101780116812b0a16810e4b045a3b2a01205b1305811134092f62072343100f0f05011e5734152612030b4c4134123009361601121781653207780a6a0d01164a25210824138107738139011481341f088158060c8133010e5920193a4c2331011a0510358101231a1b3609702732011a2f07631610033436810256174c011a1342040a58110721378139101602033067750802720201320604834303017109048244020133090481700203366f79010483520201390404810902016301048411050483420201640a048432020265310704832b02026774040481000302d5b2010259020369756c0504832902016c07048365030233700904824102016d010482670404834c0504830c02016e0104831604048120030169090481750303eba6990104835f02016f0604834c030379c2ba0a0481560201700404815e06048256020174040482370201760a04820d0201770604811c02017a0404812e06020b0202c2b9030481660301bd0a04816c0301be030483620206c99b6d7777750304835b0206ceb0646b66610402490202d8bf030482250206e8bfbc626964080482510204f09c9a9e0404830f010179010e2081335661371c01220e4e2718124f0d0649812b0b0a063b040b011402741d1235810805211a011409161d732b8106325f6a01182e330325068107703728302b011e3723081c0d0a3f810c183e061b067f0106834a12011a044030185a1e810704220a0541011245602b0e421441817801144b03811a1a29614e224b02013003048139020132050485050304f09caba6010482230201350104822708048413020138020483430201610402230201620902440108812181070301300304815e020263710404831002016403048226010483660604823c020165010484510201670104816407048418030334c2b20a0481120201690404814c02016a010481500904810f02036b75610304836402016d01027902016f0304817b03056f6373cebc0a0483010201700104817d020171050482680104843b0302383203048128040807090707070b0608060707060c0b810e07080807070707060b080707070b0807090807070a070a07070808070b06090807070708080a0b81230907070b070b0707080907070706090807070807060b07060707070808070a080b0708090b0b09810a0707060908070607060609070b0a070706080a09070707070e0a0708090b0c0b09070a080a811a0706080c09070a07080b0708060806070b080c0e07090e09060706080b060a070b0607090707130b080708070b0908070a0c810d070b0706080707080b080a0a070807090708070707090709070706080709080a81170a0707070a070707070a0b0a07080d080707060a070707070b0707060f0b060707060a08070807080708810d0807070709070b070808070907080f0b070907090b0707070a0807070c0b080c0a810107070a0b07060c07080f070b09070b0906070b070b'); INSERT INTO t1_data VALUES(274877906949,X'00000e5c033079720404826c0404833002021b03026f6b020482100201740904842e010484150303c2b36f0204840e0201760704826b02017708048273020578f48ba5b50a0481400201790904845902017a050483280304c2bac2bd0404841e0203c2aa680904843a0301bc070481100301be0304820304016a050481630202ca8103027b0202cb860604840d0204ceb56e370204832e0302bc740202520202cf8d07027e0205d8ae39c2aa0104813c0204e887b3770404816a0204f1bfb0970204832f01017a0118101e282f07045961813a193e011e69162f0d2b051c060f084460063053011e06810c20330d0733815c220515220c011210290a7e07810c3a18011a1f2f064a19155212472781047c010a123e45825501166c6062182718167131092f0112331b812c0b6e81470b01184f3a230d45261e271c36111701140a8128456b291248391a0201300104840d0204812502013401048318020137070483070201380904823b03016704026402013907023403016e060481380201610204812f0404824504045243020162050481150104824003016a0804827a020163070484590204817903017a02048209020164020484200203653077030482610302737601048424020166070481730201670304841e0504840e03016b0504825302016901048235010484400404841002016a010483680404845f02046bc2b97407023302026c7a0704812d02027161070209030378cebc0a021f020172040483750404815f0201730304813c030131020482040201740704824d0302793901027502017604048423020177040620813103048313020179030484540204833c04022503017a0904826302017a050483530203c2b2660404840f0307ba6272f397bd92010483070301be020482760202c6b9050482410203c9a3650604812c0202cbae020482180202d38c0704812f0204f096adb7070483490204f6a69c8b030484390102c2aa010c815e81147969010c811c827f0417010e81077a4c03815f010a2e8100820c010a810d148140010c0a7481201a13010c0b831525323d010a8206358129010c21637a33812701083c8340390301300504820a0304326939770404841c03016107048334030165030484150302696c0704810103016c0704812a03027178090481620301730504830e0301750504846303017802021c0301790304836803017a0604834e040135030482650302cdbf010483330304ceb23169070484090303e4849a0504817d0201b201088219744601082210832c010629846001121c8137182211816232010a81668122690108817281080110816a433581102f3e010e7481001b3481190106814e1b010e3646823a810e070301310504840b030161010481720301660604822f0301680804821003016a0704844803016b0104845b03016c0504823403026f700802340301710804826004026d3106024a03017509068458070301770a04836c04016b01021f0301790a0482790302c2b20a02270401b3040482130303ceae6e010484330302df9e040482100201b3010881656e750106820b50010e81434a27048153010a812b068122010843810c0401048211010a50817d812e010c811a8163810801061e832c010c811f81418101030163030481210301640504831102048104030482440301680a048202040271300404841303066b6576cebc6a0104843603026f7a08026d0303756a7808023e030176040483610301770704814403027973020484360302c2b906020c0401bc010484490401bd0804835c0302d38c0304846f0304eaae96750a0484020201b9010c3b824018322d010e0468315a5c817901067d583401060b810e010865118169010c0e07826b813a01066f8265010c3b8239633852010e61161e7030821b01068241210301350804843d0301620302480301690402610403e7a1910704812103016d080269030673c2bac2b36c090483070301760504821d02048454030278350702620302c2ba010483320304ca897a750804813b0401b5070482350303cebc6d050483120302d199020483020402ad660604840c0302d2a1010482550302daa30202340201ba01026d01085119826a010c2f5e82008110010882028221010481090108821b812d01068128460208810b8264010c810971812d440301310304820e0301320504825e05048221030333756d010482720301340502700301640604813d03016607020c0301690204823a03016b0104812203016c0704842103016d0304835f03016e0a04813c03016f0304834d030173060482660401710604810f0301740104843c0303d48174040482690201bc010482690108813c83290114090f816045242e148111010a810616822701048456010e2c81167a638115010a3a3b83204001067a8367010c8114127a2265010a824f7f5c230301300604833e0302616f020483560301640104827d090481280301650502150301660804841a03016804048221030169030481120301720104827803017606023d04016e020481700302c2aa0904836c0401ba090484100201bd01088240224f010843813674020a81185068620106822a44010e14154a8101825e010c19814a1b826c010c81221f81651b010a815a4d812c01082d7281160301300804843c030231300304836703013509048353030365c2bc0304814d0302667504027a0301670604831503016c09025d03016d03048178030172020483620301730104816b0402c2b30404823203017401027204016a08048451030175090237040177090481690301770302190301780a04823d03027a350a0481260302c2aa060481620401b3090481090304f098a1a30a04825d0201be010a0d730b816f010c821d81236c2b010c6f8208358119010681236801087d4e831e0108823c8235010667794b010683165e010e05812e3d3c820d010c81148123817403043531c2ba0204824c03083875c2bceba7957a03020803016302020804036dcebc0304814e0301640304843704033379790504823203016b08023603016d070481090304825804013707027a03016f080484530301710a04842203017809025e0304c2aa6135010481460401b90a04836b0401bc0704814f0306c6ab66e3afb9080482660103c39f350304821f0201b0050481110301680602030201b8040481310604810403016c0a024803017108024b0201be06048323030482650301300504812d0304f48990ae060482740103c491680a0483700301760a0481290201a7010484090202b177020482380201b3030482280102c5800602090301360104826a02018202048173040483780301370104814102018b090482730201930504810f0102c680020483790404830c020183020228020185030484400704820b02028d7a0a04821c0201920404835e02019507048437010484070301760204836102019a020214080483100303d795680504820602019e030481050201a30404820a05048436030177070484290201a50704843b0201a8060482690304816f03027735020483630202ab79050484430201b6060481050201b904024003016d070481130201bd090484350201bf06048361030163020481580102c781040484560201830604833a020286650404842a0301750a04821a020189070483370303e0b9b30904836602018c0104816c04020602021d030171050484570201960a04842d03016b0204821903016c04021e03016e0204845002019a0a04844402019d0704835103013103021702039f77750704827e0201a1090483340201a301022b0404831d0304616363750502520302cfb2040483720201a50a0481530202ad760902310201af0104811d0908816281390301640404843c0201b3080483240201bb030483480201bd040484290304840a0203bf646a0404843f03017a040482450102c89d05020f0201a1090483550201a3060483550201ab0904813203017a04024a0201ad030482210201b10204824302025d0303656577030483600202b46e010482240201b6090484410204bce39f9d020484570102c9870604813202018b0804843603017a06027102018d080481130204840c02018f0104836309026202029164020484430201930504816d0201970304843402039c6c73070483700201a0070483470201a203048262030364d7960a0484010201a80304832a060484390202a96a080481170303c69532090481440201ac040214020481180202ae770604832c0201b0090482300201b40804845a0201b5070482110302716f010482160201ba0504837b0201bb02048149030165050483250201bc07088105833002067f824d0302d19f0402330201bd04024d03017a0904814f0102ca800a048321020181040481580201820604821e020383693707048417020287630704840002018a0904834c0104835c02018b06022f02018c0704814702018d0704810a0104831a020490e7b38308048423020191020482510201920304832f030135030483710201950304833207026802019902026b02019b0904832a02019d09024c02019f0802210201a4010484080202a5710a0481680203a777670804831e0301790902100201ae0402700104834e0201af040484650202b673080484190201b705048221010483520201b808048465030167010484400104cb81cab90104834802018602048317020689777568cf920804842602018b0404817b020191050484200202a0720304833e0201a3040625827f0201a4030485080102cdb1070881408205020483700201b30404840e0202b735080482640201b80504815a0201b90a0482370206bb31cebc6c730304842f030133070483010103ce80370304826d0201900202770201ae0504844e0201b1050484080201b20904815f0201b3040483430201b40702600201b50804823b0201b90304846e0302c9a1070483300201ba0404844d0104847e0201bb050482780201bc010882378223010e811105814d8134010a8142823f2d0106811c52010a6e1814816b02088168824b010a438137812d011019060c6b812f811c010a81314e811b03033366660404825503013502048263030238620904820703016303048120040f080b0907070b07070a0907070707080a07070b0a0a81060b0707070606070f0b070b07070908070b070f0b090807080b07070707070c0e0707090d07080908080a0a50070a0707080708070706070707080a094d07070707070707070707080706070707090844070f07080c0708070708070707080a4707060609060c0b07080a07090808080737070b09060706070707070707070707094807080b0607070707060708074107080709070706070707080607060706070808070a460a0d06090709060b060707060a07070c0907060b06060b070a090707080707070b0707070c060b08070b070a09070b07070b08080706070707070807080707090d070707060707070609070a090807070d0707070b09070707070706070a0908070a0807060b0a080707090707090b08090a08070707080707070e07060708070709080b06070b0a0707070a06070606070809060a07080b07070a070c07070808070e070807070c07090607070707060707080b0743090708'); @@ -6976,7 +6975,7 @@ COMMIT; do_catchsql_test 51.1 { SELECT max(rowid)==0 FROM t1('e*'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -8752,7 +8751,7 @@ do_test 60.0 { do_catchsql_test 60.2 { SELECT (matchinfo(t1,591)) FROM t1 WHERE t1 MATCH 'e*eŸ' -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- do_test 61.0 { @@ -9773,7 +9772,7 @@ do_test 66.0 { do_catchsql_test 66.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -10107,7 +10106,7 @@ do_test 68.0 { do_catchsql_test 68.1 { PRAGMA reverse_unordered_selects=ON; INSERT INTO t1(t1) SELECT x FROM t2; -} {1 {database disk image is malformed}} +} {1 {fts5: corruption on page 1, segment 1, table "t1"}} #------------------------------------------------------------------------- reset_db @@ -10323,7 +10322,7 @@ do_test 69.0 { do_catchsql_test 69.2 { SELECT * FROM t1 WHERE a MATCH 'fx*' -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10506,7 +10505,7 @@ do_test 71.0 { do_catchsql_test 71.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10633,9 +10632,9 @@ do_catchsql_test 72.1 { INSERT INTO ttt(ttt) VALUES('integrity-check'); } {1 {database disk image is malformed}} -do_catchsql_test 72.1 { +do_catchsql_test 72.2 { SELECT 1 FROM ttt('e* NOT ee*'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10761,7 +10760,7 @@ do_test 73.0 { do_catchsql_test 73.1 { SELECT snippet(ttt,ttt, NOT 54 ), * FROM ttt('e* NOT ee*e* NOT ee* NOT ee*e* NOT e*') ; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -15910,8 +15909,220 @@ do_catchsql_test 82.4 { SAVEPOINT b; } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +reset_db +do_test 83.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-c4a4c5492615bd.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 01 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 00 4a ef fa cc 0a ad ...~.H....J..... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 00 00 00 00 00 ...m.M.+........ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 07 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 02 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 22 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 01 f3 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 b3 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 0f .............6.. +| 3168: f0 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 03 05 07 1b 88 80 80 80 80 0a 03 00 3c dd 00 00 ............<... +| 3200: 18 c2 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 90 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 0c 52 02 30 31 02 06 01 01 04 01 01 32 4...R.01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 00 f1 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 74 .....3eac......t +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 11 02 70 62 01 02 05 04 09 18 84 80 ......pb........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 0d c0 00 16 12 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 5b 01 04 04 06 1b 84 80 80 80 ..f...[......... +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 01 f8 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 00 00 00 ................ +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 02 4e 01 03 34 1e .....4t ...N..4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 19 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 60 82 16 08 04 01 2i.......2`..... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 83 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 g.z.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f 00 00 00 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ +| end crash-c4a4c5492615bd.db +}]} {} + +do_catchsql_test 83.1 { + SELECT * FROM t1('R*R*R*R*R*R*R*R*') WHERE (a,b)<=(current_date,0 BETWEEN 'a'<>11 AND '') ORDER BY rowid DESC; +} {/.*fts5: corruption found/} sqlite3_fts5_may_be_corrupt 0 finish_test - diff --git a/ext/fts5/test/fts5corrupt5.test b/ext/fts5/test/fts5corrupt5.test index 6a70fc7e4..4b21a9ff7 100644 --- a/ext/fts5/test/fts5corrupt5.test +++ b/ext/fts5/test/fts5corrupt5.test @@ -237,7 +237,7 @@ do_test 1.0 { do_catchsql_test 1.1 { SELECT * FROM t1('R*') WHERE (a,b)<=(current_date,0) ORDER BY rowid DESC; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -450,7 +450,7 @@ do_test 2.0 { do_catchsql_test 2.1 { SELECT * FROM t1('R*R*R*R*') WHERE (a,b)<=(current_date,0) ORDER BY rowid DESC; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -569,7 +569,7 @@ do_test 3.0 { do_catchsql_test 3.1 { UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thra*T'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -878,7 +878,7 @@ do_execsql_test 5.1 { } do_catchsql_test 5.4 { UPDATE t1 SET content=randomblob(500); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db diff --git a/ext/fts5/test/fts5corrupt7.test b/ext/fts5/test/fts5corrupt7.test index 41e359f42..23061a1cb 100644 --- a/ext/fts5/test/fts5corrupt7.test +++ b/ext/fts5/test/fts5corrupt7.test @@ -123,6 +123,6 @@ do_execsql_test 2.2 { do_catchsql_test 2.3 { DELETE FROM t1 WHERE rowid = 1 -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} finish_test diff --git a/ext/fts5/test/fts5corrupt8.test b/ext/fts5/test/fts5corrupt8.test index d642920e4..471a1b0e3 100644 --- a/ext/fts5/test/fts5corrupt8.test +++ b/ext/fts5/test/fts5corrupt8.test @@ -32,7 +32,7 @@ sqlite3 db test.db do_catchsql_test 1.2 { SELECT * FROM t1 -} {1 {database disk image is malformed}} +} {1 {fts5: corrupt structure record for table "t1"}} do_catchsql_test 1.3 { DROP TABLE t1 } {0 {}} @@ -90,5 +90,58 @@ do_execsql_test 3.7 { SELECT * FROM sqlite_schema } +#------------------------------------------------------------------------- +reset_db + +proc hex_to_blob {hex} { + binary encode hex $hex +} +db func hex_to_blob hex_to_blob + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1); + BEGIN; + INSERT INTO x1(rowid, x) VALUES(1, 'a b c d e f g h'); + INSERT INTO x1(rowid, x) VALUES(2, 'a b c d e f g h'); + COMMIT; + DELETE FROM x1 WHERE rowid=1; +} + +do_execsql_test 4.1 { + SELECT hex(block) FROM x1_data WHERE id=10 +} { + 00000000FF00000101010200010101010101010102 +} + +do_execsql_test 4.2.1 { + UPDATE x1_data SET block= + X'00000000FF00000101010200010101010101819C9B95A8000102' + WHERE id=10; +} + +do_catchsql_test 4.2.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + +do_execsql_test 4.3.1 { + UPDATE x1_data SET block= + X'00000000FF000001010102000101010101019282AFF9A0000102' + WHERE id=10; +} + +do_catchsql_test 4.3.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + +do_execsql_test 4.4.1 { + UPDATE x1_data SET block= + X'00000000FF000001010102000101010101018181808080130102' + WHERE id=10; +} + +do_catchsql_test 4.3.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + finish_test diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index 5c4002180..4bf120c44 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -37,6 +37,12 @@ do_execsql_test 2.1 { INSERT INTO yy(yy) VALUES('integrity-check'); } +db close +sqlite3 db test.db +do_execsql_test 2.1 { + INSERT INTO yy(yy) VALUES('integrity-check'); +} + #-------------------------------------------------------------------- # do_execsql_test 3.0 { diff --git a/ext/fts5/test/fts5leftjoin.test b/ext/fts5/test/fts5leftjoin.test index 4ef6a8961..69a172bd4 100644 --- a/ext/fts5/test/fts5leftjoin.test +++ b/ext/fts5/test/fts5leftjoin.test @@ -40,4 +40,53 @@ do_execsql_test 1.2 { SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc') } {1 abc 2 abc} + +do_execsql_test 1.3 { + DELETE FROM t1; + INSERT INTO t1 VALUES(14); +} + +do_execsql_test 1.4 { + SELECT * FROM vt LEFT JOIN t1 ON vt.rowid = 1; +} { + abc 14 + xyz {} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t0 USING fts5(a,b); + INSERT INTO t0(a,b)VALUES(1,0); + CREATE TABLE t1(x); +} + +do_execsql_test 2.1 { + SELECT * FROM t0 LEFT JOIN t1; +} {1 0 {}} + +breakpoint +do_catchsql_test 2.2 { + SELECT * FROM t0 LEFT JOIN t1 ON t0.b MATCH '1'; +} {1 {no query solution}} + +do_execsql_test 2.3 { + SELECT * FROM t0 LEFT JOIN t1 ON +b MATCH '1'; +} {1 0 {}} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t0 USING fts5(c0, c1); + INSERT INTO t0(c0,c1) VALUES (1,0); +} + +do_catchsql_test 3.1 { + SELECT * FROM t0 + LEFT JOIN ( SELECT 0 AS col_0 ) + ON ((((t0.c1 MATCH '1')AND(CASE WHEN t0.c0 THEN CAST(t0.c1 AS INTEGER) ELSE 1 END)))); +} {1 {no query solution}} + + finish_test diff --git a/ext/fts5/test/fts5rebuild.test b/ext/fts5/test/fts5rebuild.test index d74b148fb..065d16b91 100644 --- a/ext/fts5/test/fts5rebuild.test +++ b/ext/fts5/test/fts5rebuild.test @@ -46,7 +46,7 @@ do_execsql_test 1.5 { do_catchsql_test 1.6 { INSERT INTO f1(f1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_execsql_test 1.7 { INSERT INTO f1(f1) VALUES('rebuild'); diff --git a/ext/lsm1/Makefile b/ext/lsm1/Makefile deleted file mode 100644 index d497a1d13..000000000 --- a/ext/lsm1/Makefile +++ /dev/null @@ -1,56 +0,0 @@ -# -# This Makefile is designed for use with main.mk in the root directory of -# this project. After including main.mk, the users makefile should contain: -# -# LSMDIR=$(TOP)/ext/lsm1/ -# LSMOPTS=-fPIC -# include $(LSMDIR)/Makefile -# -# The most useful targets are [lsmtest] and [lsm.so]. -# - -LSMOBJ = \ - lsm_ckpt.o \ - lsm_file.o \ - lsm_log.o \ - lsm_main.o \ - lsm_mem.o \ - lsm_mutex.o \ - lsm_shared.o \ - lsm_sorted.o \ - lsm_str.o \ - lsm_tree.o \ - lsm_unix.o \ - lsm_win32.o \ - lsm_varint.o \ - lsm_vtab.o - -LSMHDR = \ - $(LSMDIR)/lsm.h \ - $(LSMDIR)/lsmInt.h - -LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c \ - $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c \ - $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c \ - $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c \ - $(LSMDIR)/lsm-test/lsmtest9.c \ - $(LSMDIR)/lsm-test/lsmtest_datasource.c \ - $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c \ - $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \ - $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \ - $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c - - -# all: lsm.so - -LSMOPTS += -fPIC -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB - -lsm.so: $(LSMOBJ) - $(T.link) -shared -fPIC -o lsm.so $(LSMOBJ) - -%.o: $(LSMDIR)/%.c $(LSMHDR) sqlite3.h - $(T.link) $(LSMOPTS) -c $< - -lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o - # $(T.link) -c $(TOP)/lsm-test/lsmtest_tdb2.cc - $(T.link) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz diff --git a/ext/lsm1/Makefile.msc b/ext/lsm1/Makefile.msc deleted file mode 100644 index 3e5a3b331..000000000 --- a/ext/lsm1/Makefile.msc +++ /dev/null @@ -1,102 +0,0 @@ -# -# This Makefile is designed for use with Makefile.msc in the root directory -# of this project. The Makefile.msc should contain: -# -# LSMDIR=$(TOP)\ext\lsm1 -# !INCLUDE $(LSMDIR)\Makefile.msc -# -# The most useful targets are [lsmtest.exe] and [lsm.dll]. -# - -LSMOBJ = \ - lsm_ckpt.lo \ - lsm_file.lo \ - lsm_log.lo \ - lsm_main.lo \ - lsm_mem.lo \ - lsm_mutex.lo \ - lsm_shared.lo \ - lsm_sorted.lo \ - lsm_str.lo \ - lsm_tree.lo \ - lsm_unix.lo \ - lsm_win32.lo \ - lsm_varint.lo \ - lsm_vtab.lo - -LSMHDR = \ - $(LSMDIR)\lsm.h \ - $(LSMDIR)\lsmInt.h - -LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c \ - $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c \ - $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c \ - $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c \ - $(LSMDIR)\lsm-test\lsmtest9.c \ - $(LSMDIR)\lsm-test\lsmtest_datasource.c \ - $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c \ - $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \ - $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \ - $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c - -# all: lsm.dll lsmtest.exe - -LSMOPTS = $(NO_WARN) -DLSM_MUTEX_WIN32=1 -I$(LSMDIR) - -!IF $(DEBUG)>2 -LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1 -!ENDIF - -!IF $(MEMDEBUG)!=0 -LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1 -!ENDIF - -lsm_ckpt.lo: $(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c - -lsm_file.lo: $(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c - -lsm_log.lo: $(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c - -lsm_main.lo: $(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c - -lsm_mem.lo: $(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c - -lsm_mutex.lo: $(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c - -lsm_shared.lo: $(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c - -lsm_sorted.lo: $(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c - -lsm_str.lo: $(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c - -lsm_tree.lo: $(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c - -lsm_unix.lo: $(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c - -lsm_win32.lo: $(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c - -lsm_varint.lo: $(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c - -lsm_vtab.lo: $(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c - -lsm.dll: $(LSMOBJ) - $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ) - copy /Y $@ $(LSMDIR)\$@ - -lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJ) - $(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJ) - copy /Y $@ $(LSMDIR)\$@ diff --git a/ext/lsm1/lsm-test/README b/ext/lsm1/lsm-test/README deleted file mode 100644 index 80654ee97..000000000 --- a/ext/lsm1/lsm-test/README +++ /dev/null @@ -1,40 +0,0 @@ - - -Organization of test case files: - - lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a - database file, then verify that the contents of the database can - be queried. - - lsmtest2.c: Crash tests. Tests that attempt to verify that the database - recovers correctly following an application or system crash. - - lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of - transactions and sub-transactions. - - lsmtest4.c: Multi-client tests. - - lsmtest5.c: Multi-client tests with a different thread for each client. - - lsmtest6.c: OOM injection tests. - - lsmtest7.c: API tests. - - lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that - the system recovers and other clients proceed unaffected if - a process fails in the middle of a write transaction. - - The difference from lsmtest2.c is that this file tests - live-recovery (recovery from a failure that occurs while other - clients are still running) whereas lsmtest2.c tests recovery - from a system or power failure. - - lsmtest9.c: More data tests. These focus on testing that calling - lsm_work(nMerge=1) to compact the database does not corrupt it. - In other words, that databases containing block-redirects - can be read and written. - - - - - diff --git a/ext/lsm1/lsm-test/lsmtest.h b/ext/lsm1/lsm-test/lsmtest.h deleted file mode 100644 index ca60424ad..000000000 --- a/ext/lsm1/lsm-test/lsmtest.h +++ /dev/null @@ -1,303 +0,0 @@ - -#ifndef __WRAPPER_INT_H_ -#define __WRAPPER_INT_H_ - -#include "lsmtest_tdb.h" -#include "sqlite3.h" -#include "lsm.h" - -#include <assert.h> -#include <stdarg.h> -#include <stdlib.h> -#include <string.h> -#include <stdio.h> -#ifndef _WIN32 -# include <unistd.h> -#endif -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <ctype.h> -#include <stdlib.h> -#include <errno.h> - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef _WIN32 -# include "windows.h" -# define gettimeofday win32GetTimeOfDay -# define F_OK (0) -# define sleep(sec) Sleep(1000 * (sec)) -# define usleep(usec) Sleep(((usec) + 999) / 1000) -# ifdef _MSC_VER -# include <io.h> -# define snprintf _snprintf -# define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) -# define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) -# define __va_copy(dst,src) ((dst) = (src)) -# define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1) -# else -# error Unsupported C compiler for Windows. -# endif -int win32GetTimeOfDay(struct timeval *, void *); -#endif - -#ifndef _LSM_INT_H -typedef unsigned int u32; -typedef unsigned char u8; -typedef long long int i64; -typedef unsigned long long int u64; -#endif - - -#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) - -#define MIN(x,y) ((x)<(y) ? (x) : (y)) -#define MAX(x,y) ((x)>(y) ? (x) : (y)) - -#define unused_parameter(x) (void)(x) - -#define TESTDB_DEFAULT_PAGE_SIZE 4096 -#define TESTDB_DEFAULT_CACHE_SIZE 2048 - -#ifndef _O_BINARY -# define _O_BINARY (0) -#endif - -/* -** Ideally, these should be in wrapper.c. But they are here instead so that -** they can be used by the C++ database wrappers in wrapper2.cc. -*/ -typedef struct DatabaseMethods DatabaseMethods; -struct TestDb { - DatabaseMethods const *pMethods; /* Database methods */ - const char *zLibrary; /* Library name for tdb_open() */ -}; -struct DatabaseMethods { - int (*xClose)(TestDb *); - int (*xWrite)(TestDb *, void *, int , void *, int); - int (*xDelete)(TestDb *, void *, int); - int (*xDeleteRange)(TestDb *, void *, int, void *, int); - int (*xFetch)(TestDb *, void *, int, void **, int *); - int (*xScan)(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) - ); - int (*xBegin)(TestDb *, int); - int (*xCommit)(TestDb *, int); - int (*xRollback)(TestDb *, int); -}; - -/* -** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the -** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but -** the primary interface is the C++ API. -*/ -int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb); -int test_kc_close(TestDb *); -int test_kc_write(TestDb *, void *, int , void *, int); -int test_kc_delete(TestDb *, void *, int); -int test_kc_delete_range(TestDb *, void *, int, void *, int); -int test_kc_fetch(TestDb *, void *, int, void **, int *); -int test_kc_scan(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) -); - -int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_mdb_close(TestDb *); -int test_mdb_write(TestDb *, void *, int , void *, int); -int test_mdb_delete(TestDb *, void *, int); -int test_mdb_fetch(TestDb *, void *, int, void **, int *); -int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) -); - -/* -** Functions in wrapper3.c. This file contains the tdb wrapper for lsm. -** The wrapper for lsm is a bit more involved than the others, as it -** includes code for a couple of different lsm configurations, and for -** various types of fault injection and robustness testing. -*/ -int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb); - -int tdb_lsm_configure(lsm_db *, const char *); - -/* Functions in lsmtest_tdb4.c */ -int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb); - - -/* Functions in testutil.c. */ -int testPrngInit(void); -u32 testPrngValue(u32 iVal); -void testPrngArray(u32 iVal, u32 *aOut, int nOut); -void testPrngString(u32 iVal, char *aOut, int nOut); - -void testErrorInit(int argc, char **); -void testPrintError(const char *zFormat, ...); -void testPrintUsage(const char *zArgs); -void testPrintFUsage(const char *zFormat, ...); -void testTimeInit(void); -int testTimeGet(void); - -/* Functions in testmem.c. */ -void testMallocInstall(lsm_env *pEnv); -void testMallocUninstall(lsm_env *pEnv); -void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *); -void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *); -void testMallocOomEnable(lsm_env *pEnv, int); - -/* lsmtest.c */ -TestDb *testOpen(const char *zSystem, int, int *pRc); -void testReopen(TestDb **ppDb, int *pRc); -void testClose(TestDb **ppDb); - -void testFetch(TestDb *, void *, int, void *, int, int *); -void testWrite(TestDb *, void *, int, void *, int, int *); -void testDelete(TestDb *, void *, int, int *); -void testDeleteRange(TestDb *, void *, int, void *, int, int *); -void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc); -void testFetchStr(TestDb *, const char *, const char *, int *pRc); - -void testBegin(TestDb *pDb, int iTrans, int *pRc); -void testCommit(TestDb *pDb, int iTrans, int *pRc); - -void test_failed(void); - -char *testMallocPrintf(const char *zFormat, ...); -char *testMallocVPrintf(const char *zFormat, va_list ap); -int testGlobMatch(const char *zPattern, const char *zStr); - -void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *); -void testFetchCompare(TestDb *, TestDb *, void *, int, int *); - -void *testMalloc(int); -void *testMallocCopy(void *pCopy, int nByte); -void *testRealloc(void *, int); -void testFree(void *); - -/* lsmtest_bt.c */ -int do_bt(int nArg, char **azArg); - -/* testio.c */ -int testVfsConfigureDb(TestDb *pDb); - -/* testfunc.c */ -int do_show(int nArg, char **azArg); -int do_work(int nArg, char **azArg); - -/* testio.c */ -int do_io(int nArg, char **azArg); - -/* lsmtest2.c */ -void do_crash_test(const char *zPattern, int *pRc); -int do_rollback_test(int nArg, char **azArg); - -/* test3.c */ -void test_rollback(const char *zSystem, const char *zPattern, int *pRc); - -/* test4.c */ -void test_mc(const char *zSystem, const char *zPattern, int *pRc); - -/* test5.c */ -void test_mt(const char *zSystem, const char *zPattern, int *pRc); - -/* lsmtest6.c */ -void test_oom(const char *zPattern, int *pRc); -void testDeleteLsmdb(const char *zFile); - -void testSaveDb(const char *zFile, const char *zAuxExt); -void testRestoreDb(const char *zFile, const char *zAuxExt); -void testCopyLsmdb(const char *zFrom, const char *zTo); - -/* lsmtest7.c */ -void test_api(const char *zPattern, int *pRc); - -/* lsmtest8.c */ -void do_writer_crash_test(const char *zPattern, int *pRc); - -/************************************************************************* -** Interface to functionality in test_datasource.c. -*/ -typedef struct Datasource Datasource; -typedef struct DatasourceDefn DatasourceDefn; - -struct DatasourceDefn { - int eType; /* A TEST_DATASOURCE_* value */ - int nMinKey; /* Minimum key size */ - int nMaxKey; /* Maximum key size */ - int nMinVal; /* Minimum value size */ - int nMaxVal; /* Maximum value size */ -}; - -#define TEST_DATASOURCE_RANDOM 1 -#define TEST_DATASOURCE_SEQUENCE 2 - -char *testDatasourceName(const DatasourceDefn *); -Datasource *testDatasourceNew(const DatasourceDefn *); -void testDatasourceFree(Datasource *); -void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *); -/* End of test_datasource.c interface. -*************************************************************************/ -void testDatasourceFetch( - TestDb *pDb, /* Database handle */ - Datasource *pData, - int iKey, - int *pRc /* IN/OUT: Error code */ -); - -void testWriteDatasource(TestDb *, Datasource *, int, int *); -void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *); -void testDeleteDatasource(TestDb *, Datasource *, int, int *); -void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *); - - -/* test1.c */ -void test_data_1(const char *, const char *, int *pRc); -void test_data_2(const char *, const char *, int *pRc); -void test_data_3(const char *, const char *, int *pRc); -void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *); -void testCaseProgress(int, int, int, int *); -int testCaseNDot(void); - -void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *); -int testControlDb(TestDb **ppDb); - -typedef struct CksumDb CksumDb; -CksumDb *testCksumArrayNew(Datasource *, int, int, int); -char *testCksumArrayGet(CksumDb *, int); -void testCksumArrayFree(CksumDb *); -void testCaseStart(int *pRc, char *zFmt, ...); -void testCaseFinish(int rc); -void testCaseSkip(void); -int testCaseBegin(int *, const char *, const char *, ...); - -#define TEST_CKSUM_BYTES 29 -int testCksumDatabase(TestDb *pDb, char *zOut); -int testCountDatabase(TestDb *pDb); -void testCompareInt(int, int, int *); -void testCompareStr(const char *z1, const char *z2, int *pRc); - -/* lsmtest9.c */ -void test_data_4(const char *, const char *, int *pRc); - - -/* -** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function. -*/ -#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z) -int testArgSelectX(void *, const char *, int, const char *, int *); - -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif - -#endif diff --git a/ext/lsm1/lsm-test/lsmtest1.c b/ext/lsm1/lsm-test/lsmtest1.c deleted file mode 100644 index 1ce2cc058..000000000 --- a/ext/lsm1/lsm-test/lsmtest1.c +++ /dev/null @@ -1,656 +0,0 @@ - -#include "lsmtest.h" - -#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE -#define DATA_RANDOM TEST_DATASOURCE_RANDOM - -typedef struct Datatest1 Datatest1; -typedef struct Datatest2 Datatest2; - -/* -** An instance of the following structure contains parameters used to -** customize the test function in this file. Test procedure: -** -** 1. Create a data-source based on the "datasource definition" vars. -** -** 2. Insert nRow key value pairs into the database. -** -** 3. Delete all keys from the database. Deletes are done in the same -** order as the inserts. -** -** During steps 2 and 3 above, after each Datatest1.nVerify inserts or -** deletes, the following: -** -** a. Run Datasource.nTest key lookups and check the results are as expected. -** -** b. If Datasource.bTestScan is true, run a handful (8) of range -** queries (scanning forwards and backwards). Check that the results -** are as expected. -** -** c. Close and reopen the database. Then run (a) and (b) again. -*/ -struct Datatest1 { - /* Datasource definition */ - DatasourceDefn defn; - - /* Test procedure parameters */ - int nRow; /* Number of rows to insert then delete */ - int nVerify; /* How often to verify the db contents */ - int nTest; /* Number of keys to test (0==all) */ - int bTestScan; /* True to do scan tests */ -}; - -/* -** An instance of the following data structure is used to describe the -** second type of test case in this file. The chief difference between -** these tests and those described by Datatest1 is that these tests also -** experiment with range-delete operations. Tests proceed as follows: -** -** 1. Open the datasource described by Datatest2.defn. -** -** 2. Open a connection on an empty database. -** -** 3. Do this Datatest2.nIter times: -** -** a) Insert Datatest2.nWrite key-value pairs from the datasource. -** -** b) Select two pseudo-random keys and use them as the start -** and end points of a range-delete operation. -** -** c) Verify that the contents of the database are as expected (see -** below for details). -** -** d) Close and then reopen the database handle. -** -** e) Verify that the contents of the database are still as expected. -** -** The inserts and range deletes are run twice - once on the database being -** tested and once using a control system (sqlite3, kc etc. - something that -** works). In order to verify that the contents of the db being tested are -** correct, the test runs a bunch of scans and lookups on both the test and -** control databases. If the results are the same, the test passes. -*/ -struct Datatest2 { - DatasourceDefn defn; - int nRange; - int nWrite; /* Number of writes per iteration */ - int nIter; /* Total number of iterations to run */ -}; - -/* -** Generate a unique name for the test case pTest with database system -** zSystem. -*/ -static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", - zSystem, zData, bRecover, pTest->nRow, pTest->nVerify - ); - testFree(zData); - return zRet; -} - -int testControlDb(TestDb **ppDb){ -#ifdef HAVE_KYOTOCABINET - return tdb_open("kyotocabinet", "tmp.db", 1, ppDb); -#else - return tdb_open("sqlite3", "", 1, ppDb); -#endif -} - -void testDatasourceFetch( - TestDb *pDb, /* Database handle */ - Datasource *pData, - int iKey, - int *pRc /* IN/OUT: Error code */ -){ - void *pKey; int nKey; /* Database key to query for */ - void *pVal; int nVal; /* Expected result of query */ - - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testFetch(pDb, pKey, nKey, pVal, nVal, pRc); -} - -/* -** This function is called to test that the contents of database pDb -** are as expected. In this case, expected is defined as containing -** key-value pairs iFirst through iLast, inclusive, from data source -** pData. In other words, a loop like the following could be used to -** construct a database with identical contents from scratch. -** -** for(i=iFirst; i<=iLast; i++){ -** testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); -** // insert (pKey, nKey) -> (pVal, nVal) into database -** } -** -** The key domain consists of keys 0 to (nRow-1), inclusive, from -** data source pData. For both scan and lookup tests, keys are selected -** pseudo-randomly from within this set. -** -** This function runs nLookupTest lookup tests and nScanTest scan tests. -** -** A lookup test consists of selecting a key from the domain and querying -** pDb for it. The test fails if the presence of the key and, if present, -** the associated value do not match the expectations defined above. -** -** A scan test involves selecting a key from the domain and running -** the following queries: -** -** 1. Scan all keys equal to or greater than the key, in ascending order. -** 2. Scan all keys equal to or smaller than the key, in descending order. -** -** Additionally, if nLookupTest is greater than zero, the following are -** run once: -** -** 1. Scan all keys in the db, in ascending order. -** 2. Scan all keys in the db, in descending order. -** -** As you would assume, the test fails if the returned values do not match -** expectations. -*/ -void testDbContents( - TestDb *pDb, /* Database handle being tested */ - Datasource *pData, /* pDb contains data from here */ - int nRow, /* Size of key domain */ - int iFirst, /* Index of first key from pData in pDb */ - int iLast, /* Index of last key from pData in pDb */ - int nLookupTest, /* Number of lookup tests to run */ - int nScanTest, /* Number of scan tests to run */ - int *pRc /* IN/OUT: Error code */ -){ - int j; - int rc = *pRc; - - if( rc==0 && nScanTest ){ - TestDb *pDb2 = 0; - - /* Open a control db (i.e. one that we assume works) */ - rc = testControlDb(&pDb2); - - for(j=iFirst; rc==0 && j<=iLast; j++){ - void *pKey; int nKey; /* Database key to insert */ - void *pVal; int nVal; /* Database value to insert */ - testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal); - rc = tdb_write(pDb2, pKey, nKey, pVal, nVal); - } - - if( rc==0 ){ - int iKey1; - int iKey2; - void *pKey1; int nKey1; /* Start key */ - void *pKey2; int nKey2; /* Final key */ - - iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow; - iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow; - testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0); - pKey1 = testMalloc(nKey1+1); - memcpy(pKey1, pKey2, nKey1+1); - testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0); - - testScanCompare(pDb2, pDb, 0, 0, 0, 0, 0, &rc); - testScanCompare(pDb2, pDb, 0, 0, 0, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0, &rc); - testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 1, 0, 0, 0, 0, &rc); - testScanCompare(pDb2, pDb, 1, 0, 0, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0, &rc); - testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc); - testFree(pKey1); - } - tdb_close(pDb2); - } - - /* Test some lookups. */ - for(j=0; rc==0 && j<nLookupTest; j++){ - int iKey; /* Datasource key to test */ - void *pKey; int nKey; /* Database key to query for */ - void *pVal; int nVal; /* Expected result of query */ - - if( nLookupTest>=nRow ){ - iKey = j; - }else{ - iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow; - } - - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - if( iFirst>iKey || iKey>iLast ){ - pVal = 0; - nVal = -1; - } - - testFetch(pDb, pKey, nKey, pVal, nVal, &rc); - } - - *pRc = rc; -} - -/* -** This function should be called during long running test cases to output -** the progress dots (...) to stdout. -*/ -void testCaseProgress(int i, int n, int nDot, int *piDot){ - int iDot = *piDot; - while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){ - printf("."); - fflush(stdout); - iDot++; - } - *piDot = iDot; -} - -int testCaseNDot(void){ return 20; } - -#if 0 -static void printScanCb( - void *pCtx, void *pKey, int nKey, void *pVal, int nVal -){ - printf("%s\n", (char *)pKey); - fflush(stdout); -} -#endif - -void testReopenRecover(TestDb **ppDb, int *pRc){ - if( *pRc==0 ){ - const char *zLib = tdb_library_name(*ppDb); - const char *zDflt = tdb_default_db(zLib); - testCopyLsmdb(zDflt, "bak.db"); - testClose(ppDb); - testCopyLsmdb("bak.db", zDflt); - *pRc = tdb_open(zLib, 0, 0, ppDb); - } -} - - -static void doDataTest1( - const char *zSystem, /* Database system to test */ - int bRecover, - Datatest1 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - int i; - int iDot; - int rc = LSM_OK; - Datasource *pData; - TestDb *pDb; - int iToggle = 0; - - /* Start the test case, open a database and allocate the datasource. */ - pDb = testOpen(zSystem, 1, &rc); - pData = testDatasourceNew(&p->defn); - - i = 0; - iDot = 0; - while( rc==LSM_OK && i<p->nRow ){ - - /* Insert some data */ - testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc); - i += p->nVerify; - - if( iToggle ) testBegin(pDb, 1, &rc); - /* Check that the db content is correct. */ - testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); - if( iToggle ) testCommit(pDb, 0, &rc); - iToggle = (iToggle+1)%2; - - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - - /* Check that the db content is still correct. */ - testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); - } - - i = 0; - iDot = 0; - while( rc==LSM_OK && i<p->nRow ){ - - /* Delete some entries */ - testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc); - i += p->nVerify; - - /* Check that the db content is correct. */ - testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); - - /* Close and reopen the database. */ - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - - /* Check that the db content is still correct. */ - testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); - } - - /* Free the datasource, close the database and finish the test case. */ - testDatasourceFree(pData); - tdb_close(pDb); - testCaseFinish(rc); - *pRc = rc; -} - - -void test_data_1( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest1 aTest[] = { - { {DATA_RANDOM, 500,600, 1000,2000}, 1000, 100, 10, 0}, - { {DATA_RANDOM, 20,25, 100,200}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 100,200}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 10,20}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 1000,2000}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,100, 10000,20000}, 100, 25, 100, 1}, - { {DATA_RANDOM, 80,100, 10,20}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 5000,6000, 10,20}, 100, 25, 100, 1}, - { {DATA_SEQUENTIAL, 5,10, 10,20}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,10, 100,200}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,10, 1000,2000}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,100, 10000,20000}, 100, 25, 100, 1}, - { {DATA_RANDOM, 10,10, 100,100}, 100000, 1000, 100, 0}, - { {DATA_SEQUENTIAL, 10,10, 100,100}, 100000, 1000, 100, 0}, - }; - - int i; - int bRecover; - - for(bRecover=0; bRecover<2; bRecover++){ - if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; - for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ - char *zName = getName(zSystem, bRecover, &aTest[i]); - if( testCaseBegin(pRc, zPattern, "%s", zName) ){ - doDataTest1(zSystem, bRecover, &aTest[i], pRc); - } - testFree(zName); - } - } -} - -void testCompareDb( - Datasource *pData, - int nData, - int iSeed, - TestDb *pControl, - TestDb *pDb, - int *pRc -){ - int i; - - static int nCall = 0; - nCall++; - - testScanCompare(pControl, pDb, 0, 0, 0, 0, 0, pRc); - testScanCompare(pControl, pDb, 1, 0, 0, 0, 0, pRc); - - if( *pRc==0 ){ - int iKey1; - int iKey2; - void *pKey1; int nKey1; /* Start key */ - void *pKey2; int nKey2; /* Final key */ - - iKey1 = testPrngValue(iSeed) % nData; - iKey2 = testPrngValue(iSeed+1) % nData; - testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0); - pKey1 = testMalloc(nKey1+1); - memcpy(pKey1, pKey2, nKey1+1); - testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0); - - testScanCompare(pControl, pDb, 0, 0, 0, pKey2, nKey2, pRc); - testScanCompare(pControl, pDb, 0, pKey1, nKey1, 0, 0, pRc); - testScanCompare(pControl, pDb, 0, pKey1, nKey1, pKey2, nKey2, pRc); - testScanCompare(pControl, pDb, 1, 0, 0, pKey2, nKey2, pRc); - testScanCompare(pControl, pDb, 1, pKey1, nKey1, 0, 0, pRc); - testScanCompare(pControl, pDb, 1, pKey1, nKey1, pKey2, nKey2, pRc); - testFree(pKey1); - } - - for(i=0; i<nData && *pRc==0; i++){ - void *pKey; int nKey; - testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0); - testFetchCompare(pControl, pDb, pKey, nKey, pRc); - } -} - -static void doDataTest2( - const char *zSystem, /* Database system to test */ - int bRecover, - Datatest2 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - TestDb *pDb; - TestDb *pControl; - Datasource *pData; - int i; - int rc = LSM_OK; - int iDot = 0; - - /* Start the test case, open a database and allocate the datasource. */ - pDb = testOpen(zSystem, 1, &rc); - pData = testDatasourceNew(&p->defn); - rc = testControlDb(&pControl); - - if( tdb_lsm(pDb) ){ - int nBuf = 32 * 1024 * 1024; - lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf); - } - - for(i=0; rc==0 && i<p->nIter; i++){ - void *pKey1; int nKey1; - void *pKey2; int nKey2; - int ii; - int nRange = MIN(p->nIter*p->nWrite, p->nRange); - - for(ii=0; rc==0 && ii<p->nWrite; ii++){ - int iKey = (i*p->nWrite + ii) % p->nRange; - testWriteDatasource(pControl, pData, iKey, &rc); - testWriteDatasource(pDb, pData, iKey, &rc); - } - - testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0); - pKey1 = testMallocCopy(pKey1, nKey1); - testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0); - - testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc); - testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc); - testFree(pKey1); - - testCompareDb(pData, nRange, i, pControl, pDb, &rc); - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - testCompareDb(pData, nRange, i, pControl, pDb, &rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); - } - - testClose(&pDb); - testClose(&pControl); - testDatasourceFree(pData); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", - zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter - ); - testFree(zData); - return zRet; -} - -void test_data_2( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest2 aTest[] = { - /* defn, nRange, nWrite, nIter */ - { {DATA_RANDOM, 20,25, 100,200}, 10000, 10, 50 }, - { {DATA_RANDOM, 20,25, 100,200}, 10000, 200, 50 }, - { {DATA_RANDOM, 20,25, 100,200}, 100, 10, 1000 }, - { {DATA_RANDOM, 20,25, 100,200}, 100, 200, 50 }, - }; - - int i; - int bRecover; - - for(bRecover=0; bRecover<2; bRecover++){ - if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; - for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ - char *zName = getName2(zSystem, bRecover, &aTest[i]); - if( testCaseBegin(pRc, zPattern, "%s", zName) ){ - doDataTest2(zSystem, bRecover, &aTest[i], pRc); - } - testFree(zName); - } - } -} - -/************************************************************************* -** Test case data3.* -*/ - -typedef struct Datatest3 Datatest3; -struct Datatest3 { - int nRange; /* Keys are between 1 and this value, incl. */ - int nIter; /* Number of iterations */ - int nWrite; /* Number of writes per iteration */ - int nDelete; /* Number of deletes per iteration */ - - int nValMin; /* Minimum value size for writes */ - int nValMax; /* Maximum value size for writes */ -}; - -void testPutU32(u8 *aBuf, u32 iVal){ - aBuf[0] = (iVal >> 24) & 0xFF; - aBuf[1] = (iVal >> 16) & 0xFF; - aBuf[2] = (iVal >> 8) & 0xFF; - aBuf[3] = (iVal >> 0) & 0xFF; -} - -void dt3PutKey(u8 *aBuf, int iKey){ - assert( iKey<100000 && iKey>=0 ); - sprintf((char *)aBuf, "%.5d", iKey); -} - -static void doDataTest3( - const char *zSystem, /* Database system to test */ - Datatest3 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - int iDot = 0; - int rc = *pRc; - TestDb *pDb; - u8 *abPresent; /* Array of boolean */ - char *aVal; /* Buffer to hold values */ - int i; - u32 iSeq = 10; /* prng counter */ - - abPresent = (u8 *)testMalloc(p->nRange+1); - aVal = (char *)testMalloc(p->nValMax+1); - pDb = testOpen(zSystem, 1, &rc); - - for(i=0; i<p->nIter && rc==0; i++){ - int ii; - - testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); - - /* Perform nWrite inserts */ - for(ii=0; ii<p->nWrite; ii++){ - u8 aKey[6]; - u32 iKey; - int nVal; - - iKey = (testPrngValue(iSeq++) % p->nRange) + 1; - nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin; - testPrngString(testPrngValue(iSeq++), aVal, nVal); - dt3PutKey(aKey, iKey); - - testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc); - abPresent[iKey] = 1; - } - - /* Perform nDelete deletes */ - for(ii=0; ii<p->nDelete; ii++){ - u8 aKey1[6]; - u8 aKey2[6]; - u32 iKey; - - iKey = (testPrngValue(iSeq++) % p->nRange) + 1; - dt3PutKey(aKey1, iKey-1); - dt3PutKey(aKey2, iKey+1); - - testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc); - abPresent[iKey] = 0; - } - - testReopen(&pDb, &rc); - - for(ii=1; rc==0 && ii<=p->nRange; ii++){ - int nDbVal; - void *pDbVal; - u8 aKey[6]; - int dbrc; - - dt3PutKey(aKey, ii); - dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal); - testCompareInt(0, dbrc, &rc); - - if( abPresent[ii] ){ - testCompareInt(1, (nDbVal>0), &rc); - }else{ - testCompareInt(1, (nDbVal<0), &rc); - } - } - } - - testClose(&pDb); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName3(const char *zSystem, Datatest3 *p){ - return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)", - zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, - p->nValMin, p->nValMax - ); -} - -void test_data_3( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest3 aTest[] = { - /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */ - { 100, 1000, 5, 5, 50, 100 }, - { 100, 1000, 2, 2, 5, 10 }, - }; - - int i; - - for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ - char *zName = getName3(zSystem, &aTest[i]); - if( testCaseBegin(pRc, zPattern, "%s", zName) ){ - doDataTest3(zSystem, &aTest[i], pRc); - } - testFree(zName); - } -} diff --git a/ext/lsm1/lsm-test/lsmtest2.c b/ext/lsm1/lsm-test/lsmtest2.c deleted file mode 100644 index d2ef0eb36..000000000 --- a/ext/lsm1/lsm-test/lsmtest2.c +++ /dev/null @@ -1,488 +0,0 @@ - -/* -** This file contains tests related to recovery following application -** and system crashes (power failures) while writing to the database. -*/ - -#include "lsmtest.h" - -/* -** Structure used by testCksumDatabase() to accumulate checksum values in. -*/ -typedef struct Cksum Cksum; -struct Cksum { - int nRow; - int cksum1; - int cksum2; -}; - -/* -** tdb_scan() callback used by testCksumDatabase() -*/ -static void scanCksumDb( - void *pCtx, - void *pKey, int nKey, - void *pVal, int nVal -){ - Cksum *p = (Cksum *)pCtx; - int i; - - p->nRow++; - for(i=0; i<nKey; i++){ - p->cksum1 += ((u8 *)pKey)[i]; - p->cksum2 += p->cksum1; - } - for(i=0; i<nVal; i++){ - p->cksum1 += ((u8 *)pVal)[i]; - p->cksum2 += p->cksum1; - } -} - -/* -** tdb_scan() callback used by testCountDatabase() -*/ -static void scanCountDb( - void *pCtx, - void *pKey, int nKey, - void *pVal, int nVal -){ - Cksum *p = (Cksum *)pCtx; - p->nRow++; - - unused_parameter(pKey); - unused_parameter(nKey); - unused_parameter(pVal); - unused_parameter(nVal); -} - - -/* -** Iterate through the entire contents of database pDb. Write a checksum -** string based on the db contents into buffer zOut before returning. A -** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size: -** -** * 32-bit integer (10 bytes) -** * 1 space (1 byte) -** * 32-bit hex (8 bytes) -** * 1 space (1 byte) -** * 32-bit hex (8 bytes) -** * nul-terminator (1 byte) -** -** The number of entries in the database is returned. -*/ -int testCksumDatabase( - TestDb *pDb, /* Database handle */ - char *zOut /* Buffer to write checksum to */ -){ - Cksum cksum; - memset(&cksum, 0, sizeof(Cksum)); - tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb); - sprintf(zOut, "%d %x %x", - cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2 - ); - assert( strlen(zOut)<TEST_CKSUM_BYTES ); - return cksum.nRow; -} - -int testCountDatabase(TestDb *pDb){ - Cksum cksum; - memset(&cksum, 0, sizeof(Cksum)); - tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCountDb); - return cksum.nRow; -} - -/* -** This function is a no-op if *pRc is not 0 when it is called. -** -** Otherwise, the two nul-terminated strings z1 and z1 are compared. If -** they are the same, the function returns without doing anything. Otherwise, -** an error message is printed, *pRc is set to 1 and the test_failed() -** function called. -*/ -void testCompareStr(const char *z1, const char *z2, int *pRc){ - if( *pRc==0 ){ - if( strcmp(z1, z2) ){ - testPrintError("testCompareStr: \"%s\" != \"%s\"\n", z1, z2); - *pRc = 1; - test_failed(); - } - } -} - -/* -** This function is a no-op if *pRc is not 0 when it is called. -** -** Otherwise, the two integers i1 and i2 are compared. If they are equal, -** the function returns without doing anything. Otherwise, an error message -** is printed, *pRc is set to 1 and the test_failed() function called. -*/ -void testCompareInt(int i1, int i2, int *pRc){ - if( *pRc==0 && i1!=i2 ){ - testPrintError("testCompareInt: %d != %d\n", i1, i2); - *pRc = 1; - test_failed(); - } -} - -void testCaseStart(int *pRc, char *zFmt, ...){ - va_list ap; - va_start(ap, zFmt); - vprintf(zFmt, ap); - printf(" ..."); - va_end(ap); - *pRc = 0; - fflush(stdout); -} - -/* -** This function is a no-op if *pRc is non-zero when it is called. Zero -** is returned in this case. -** -** Otherwise, the zFmt (a printf style format string) and following arguments -** are used to create a test case name. If zPattern is NULL or a glob pattern -** that matches the test case name, 1 is returned and the test case started. -** Otherwise, zero is returned and the test case does not start. -*/ -int testCaseBegin(int *pRc, const char *zPattern, const char *zFmt, ...){ - int res = 0; - if( *pRc==0 ){ - char *zTest; - va_list ap; - - va_start(ap, zFmt); - zTest = testMallocVPrintf(zFmt, ap); - va_end(ap); - if( zPattern==0 || testGlobMatch(zPattern, zTest) ){ - printf("%-50s ...", zTest); - res = 1; - } - testFree(zTest); - fflush(stdout); - } - - return res; -} - -void testCaseFinish(int rc){ - if( rc==0 ){ - printf("Ok\n"); - }else{ - printf("FAILED\n"); - } - fflush(stdout); -} - -void testCaseSkip(){ - printf("Skipped\n"); -} - -void testSetupSavedLsmdb( - const char *zCfg, - const char *zFile, - Datasource *pData, - int nRow, - int *pRc -){ - if( *pRc==0 ){ - int rc; - TestDb *pDb; - rc = tdb_lsm_open(zCfg, zFile, 1, &pDb); - if( rc==0 ){ - testWriteDatasourceRange(pDb, pData, 0, nRow, &rc); - testClose(&pDb); - if( rc==0 ) testSaveDb(zFile, "log"); - } - *pRc = rc; - } -} - -/* -** This function is a no-op if *pRc is non-zero when it is called. -** -** Open the LSM database identified by zFile and compute its checksum -** (a string, as returned by testCksumDatabase()). If the checksum is -** identical to zExpect1 or, if it is not NULL, zExpect2, the test passes. -** Otherwise, print an error message and set *pRc to 1. -*/ -static void testCompareCksumLsmdb( - const char *zFile, /* Path to LSM database */ - int bCompress, /* True if db is compressed */ - const char *zExpect1, /* Expected checksum 1 */ - const char *zExpect2, /* Expected checksum 2 (or NULL) */ - int *pRc /* IN/OUT: Test case error code */ -){ - if( *pRc==0 ){ - char zCksum[TEST_CKSUM_BYTES]; - TestDb *pDb; - - *pRc = tdb_lsm_open((bCompress?"compression=1 mmap=0":""), zFile, 0, &pDb); - testCksumDatabase(pDb, zCksum); - testClose(&pDb); - - if( *pRc==0 ){ - int r1 = 0; - int r2 = -1; - - r1 = strcmp(zCksum, zExpect1); - if( zExpect2 ) r2 = strcmp(zCksum, zExpect2); - if( r1 && r2 ){ - if( zExpect2 ){ - testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")", - zCksum, zExpect1, zExpect2 - ); - }else{ - testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"", - zCksum, zExpect1 - ); - } - *pRc = 1; - test_failed(); - } - } - } -} - -#if 0 /* not used */ -static void testCompareCksumBtdb( - const char *zFile, /* Path to LSM database */ - const char *zExpect1, /* Expected checksum 1 */ - const char *zExpect2, /* Expected checksum 2 (or NULL) */ - int *pRc /* IN/OUT: Test case error code */ -){ - if( *pRc==0 ){ - char zCksum[TEST_CKSUM_BYTES]; - TestDb *pDb; - - *pRc = tdb_open("bt", zFile, 0, &pDb); - testCksumDatabase(pDb, zCksum); - testClose(&pDb); - - if( *pRc==0 ){ - int r1 = 0; - int r2 = -1; - - r1 = strcmp(zCksum, zExpect1); - if( zExpect2 ) r2 = strcmp(zCksum, zExpect2); - if( r1 && r2 ){ - if( zExpect2 ){ - testPrintError("testCompareCksumLsmdb: \"%s\" != (\"%s\" OR \"%s\")", - zCksum, zExpect1, zExpect2 - ); - }else{ - testPrintError("testCompareCksumLsmdb: \"%s\" != \"%s\"", - zCksum, zExpect1 - ); - } - *pRc = 1; - test_failed(); - } - } - } -} -#endif /* not used */ - -/* Above this point are reusable test routines. Not clear that they -** should really be in this file. -*************************************************************************/ - -/* -** This test verifies that if a system crash occurs while doing merge work -** on the db, no data is lost. -*/ -static void crash_test1(int bCompress, int *pRc){ - const char *DBNAME = "testdb.lsm"; - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 200, 200}; - - const int nRow = 5000; /* Database size */ - const int nIter = 200; /* Number of test iterations */ - const int nWork = 20; /* Maximum lsm_work() calls per iteration */ - const int nPage = 15; /* Pages per lsm_work call */ - - int i; - int iDot = 0; - Datasource *pData; - CksumDb *pCksumDb; - TestDb *pDb; - char *zCfg; - - const char *azConfig[2] = { - "page_size=1024 block_size=65536 autoflush=16384 safety=2 mmap=0", - "page_size=1024 block_size=65536 autoflush=16384 safety=2 " - " compression=1 mmap=0" - }; - assert( bCompress==0 || bCompress==1 ); - - /* Allocate datasource. And calculate the expected checksums. */ - pData = testDatasourceNew(&defn); - pCksumDb = testCksumArrayNew(pData, nRow, nRow, 1); - - /* Setup and save the initial database. */ - - zCfg = testMallocPrintf("%s automerge=7", azConfig[bCompress]); - testSetupSavedLsmdb(zCfg, DBNAME, pData, 5000, pRc); - testFree(zCfg); - - for(i=0; i<nIter && *pRc==0; i++){ - int iWork; - int testrc = 0; - - testCaseProgress(i, nIter, testCaseNDot(), &iDot); - - /* Restore and open the database. */ - testRestoreDb(DBNAME, "log"); - testrc = tdb_lsm_open(azConfig[bCompress], DBNAME, 0, &pDb); - assert( testrc==0 ); - - /* Call lsm_work() on the db */ - tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nWork*2))); - for(iWork=0; testrc==0 && iWork<nWork; iWork++){ - int nWrite = 0; - lsm_db *db = tdb_lsm(pDb); - testrc = lsm_work(db, 0, nPage, &nWrite); - /* assert( testrc!=0 || nWrite>0 ); */ - if( testrc==0 ) testrc = lsm_checkpoint(db, 0); - } - tdb_close(pDb); - - /* Check that the database content is still correct */ - testCompareCksumLsmdb(DBNAME, - bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc); - } - - testCksumArrayFree(pCksumDb); - testDatasourceFree(pData); -} - -/* -** This test verifies that if a system crash occurs while committing a -** transaction to the log file, no earlier transactions are lost or damaged. -*/ -static void crash_test2(int bCompress, int *pRc){ - const char *DBNAME = "testdb.lsm"; - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000}; - - const int nIter = 200; - const int nInsert = 20; - - int i; - int iDot = 0; - Datasource *pData; - CksumDb *pCksumDb; - TestDb *pDb; - - /* Allocate datasource. And calculate the expected checksums. */ - pData = testDatasourceNew(&defn); - pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1); - - /* Setup and save the initial database. */ - testSetupSavedLsmdb("", DBNAME, pData, 100, pRc); - - for(i=0; i<nIter && *pRc==0; i++){ - int iIns; - int testrc = 0; - - testCaseProgress(i, nIter, testCaseNDot(), &iDot); - - /* Restore and open the database. */ - testRestoreDb(DBNAME, "log"); - testrc = tdb_lsm_open("safety=2", DBNAME, 0, &pDb); - assert( testrc==0 ); - - /* Insert nInsert records into the database. Crash midway through. */ - tdb_lsm_prepare_sync_crash(pDb, 1 + (i%(nInsert+2))); - for(iIns=0; iIns<nInsert; iIns++){ - void *pKey; int nKey; - void *pVal; int nVal; - - testDatasourceEntry(pData, 100+iIns, &pKey, &nKey, &pVal, &nVal); - testrc = tdb_write(pDb, pKey, nKey, pVal, nVal); - if( testrc ) break; - } - tdb_close(pDb); - - /* Check that no data was lost when the system crashed. */ - testCompareCksumLsmdb(DBNAME, bCompress, - testCksumArrayGet(pCksumDb, 100 + iIns), - testCksumArrayGet(pCksumDb, 100 + iIns + 1), - pRc - ); - } - - testDatasourceFree(pData); - testCksumArrayFree(pCksumDb); -} - - -/* -** This test verifies that if a system crash occurs when checkpointing -** the database, data is not lost (assuming that any writes not synced -** to the db have been synced into the log file). -*/ -static void crash_test3(int bCompress, int *pRc){ - const char *DBNAME = "testdb.lsm"; - const int nIter = 100; - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000}; - - int i; - int iDot = 0; - Datasource *pData; - CksumDb *pCksumDb; - TestDb *pDb; - - /* Allocate datasource. And calculate the expected checksums. */ - pData = testDatasourceNew(&defn); - pCksumDb = testCksumArrayNew(pData, 110, 150, 10); - - /* Setup and save the initial database. */ - testSetupSavedLsmdb("", DBNAME, pData, 100, pRc); - - for(i=0; i<nIter && *pRc==0; i++){ - int iOpen; - testCaseProgress(i, nIter, testCaseNDot(), &iDot); - testRestoreDb(DBNAME, "log"); - - for(iOpen=0; iOpen<5; iOpen++){ - /* Open the database. Insert 10 more records. */ - pDb = testOpen("lsm", 0, pRc); - testWriteDatasourceRange(pDb, pData, 100+iOpen*10, 10, pRc); - - /* Schedule a crash simulation then close the db. */ - tdb_lsm_prepare_sync_crash(pDb, 1 + (i%2)); - tdb_close(pDb); - - /* Open the database and check that the crash did not cause any - ** data loss. */ - testCompareCksumLsmdb(DBNAME, bCompress, - testCksumArrayGet(pCksumDb, 110 + iOpen*10), 0, - pRc - ); - } - } - - testDatasourceFree(pData); - testCksumArrayFree(pCksumDb); -} - -void do_crash_test(const char *zPattern, int *pRc){ - struct Test { - const char *zTest; - void (*x)(int, int *); - int bCompress; - } aTest [] = { - { "crash.lsm.1", crash_test1, 0 }, -#ifdef HAVE_ZLIB - { "crash.lsm_zip.1", crash_test1, 1 }, -#endif - { "crash.lsm.2", crash_test2, 0 }, - { "crash.lsm.3", crash_test3, 0 }, - }; - int i; - - for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ - struct Test *p = &aTest[i]; - if( testCaseBegin(pRc, zPattern, "%s", p->zTest) ){ - p->x(p->bCompress, pRc); - testCaseFinish(*pRc); - } - } -} diff --git a/ext/lsm1/lsm-test/lsmtest3.c b/ext/lsm1/lsm-test/lsmtest3.c deleted file mode 100644 index 760dec300..000000000 --- a/ext/lsm1/lsm-test/lsmtest3.c +++ /dev/null @@ -1,238 +0,0 @@ - - -/* -** This file contains tests related to the explicit rollback of database -** transactions and sub-transactions. -*/ - - -/* -** Repeat 2000 times (until the db contains 100,000 entries): -** -** 1. Open a transaction and insert 500 rows, opening a nested -** sub-transaction each 100 rows. -** -** 2. Roll back to each sub-transaction savepoint. Check the database -** checksum looks Ok. -** -** 3. Every second iteration, roll back the main transaction. Check the -** db checksum is correct. Every other iteration, commit the main -** transaction (increasing the size of the db by 100 rows). -*/ - - -#include "lsmtest.h" - -struct CksumDb { - int nFirst; - int nLast; - int nStep; - char **azCksum; -}; - -CksumDb *testCksumArrayNew( - Datasource *pData, - int nFirst, - int nLast, - int nStep -){ - TestDb *pDb; - CksumDb *pRet; - int i; - int nEntry; - int rc = 0; - - assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 ); - - pRet = malloc(sizeof(CksumDb)); - memset(pRet, 0, sizeof(CksumDb)); - pRet->nFirst = nFirst; - pRet->nLast = nLast; - pRet->nStep = nStep; - nEntry = 1 + ((nLast - nFirst) / nStep); - - /* Allocate space so that azCksum is an array of nEntry pointers to - ** buffers each TEST_CKSUM_BYTES in size. */ - pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES)); - for(i=0; i<nEntry; i++){ - char *pStart = (char *)(&pRet->azCksum[nEntry]); - pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES]; - } - - tdb_open("lsm", "tempdb.lsm", 1, &pDb); - testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc); - for(i=0; i<nEntry; i++){ - testCksumDatabase(pDb, pRet->azCksum[i]); - if( i==nEntry ) break; - testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc); - } - - tdb_close(pDb); - - return pRet; -} - -char *testCksumArrayGet(CksumDb *p, int nRow){ - int i; - assert( nRow>=p->nFirst ); - assert( nRow<=p->nLast ); - assert( ((nRow-p->nFirst) % p->nStep)==0 ); - - i = (nRow - p->nFirst) / p->nStep; - return p->azCksum[i]; -} - -void testCksumArrayFree(CksumDb *p){ - free(p->azCksum); - memset(p, 0x55, sizeof(*p)); - free(p); -} - -/* End of CksumDb code. -**************************************************************************/ - -/* -** Test utility function. Write key-value pair $i from datasource pData -** into database pDb. -*/ -void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); - testWrite(pDb, pKey, nKey, pVal, nVal, pRc); -} - -/* -** Test utility function. Delete datasource pData key $i from database pDb. -*/ -void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ - void *pKey; int nKey; - testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0); - testDelete(pDb, pKey, nKey, pRc); -} - -/* -** This function inserts nWrite key/value pairs into database pDb - the -** nWrite key value pairs starting at iFirst from data source pData. -*/ -void testWriteDatasourceRange( - TestDb *pDb, /* Database to write to */ - Datasource *pData, /* Data source to read values from */ - int iFirst, /* Index of first key/value pair */ - int nWrite, /* Number of key/value pairs to write */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - for(i=0; i<nWrite; i++){ - testWriteDatasource(pDb, pData, iFirst+i, pRc); - } -} - -void testDeleteDatasourceRange( - TestDb *pDb, /* Database to write to */ - Datasource *pData, /* Data source to read keys from */ - int iFirst, /* Index of first key */ - int nWrite, /* Number of keys to delete */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - for(i=0; i<nWrite; i++){ - testDeleteDatasource(pDb, pData, iFirst+i, pRc); - } -} - -static char *getName(const char *zSystem){ - char *zRet; - zRet = testMallocPrintf("rollback.%s", zSystem); - return zRet; -} - -static int rollback_test_1( - const char *zSystem, - Datasource *pData -){ - const int nRepeat = 100; - - TestDb *pDb; - int rc; - int i; - CksumDb *pCksum; - char *zName; - - zName = getName(zSystem); - testCaseStart(&rc, zName); - testFree(zName); - - pCksum = testCksumArrayNew(pData, 0, nRepeat*100, 100); - pDb = 0; - rc = tdb_open(zSystem, 0, 1, &pDb); - if( pDb && tdb_transaction_support(pDb)==0 ){ - testCaseSkip(); - goto skip_rollback_test; - } - - for(i=0; i<nRepeat && rc==0; i++){ - char zCksum[TEST_CKSUM_BYTES]; - int nCurrent = (((i+1)/2) * 100); - int nDbRow; - int iTrans; - - /* Check that the database is the expected size. */ - nDbRow = testCountDatabase(pDb); - testCompareInt(nCurrent, nDbRow, &rc); - - for(iTrans=2; iTrans<=6 && rc==0; iTrans++){ - tdb_begin(pDb, iTrans); - testWriteDatasourceRange(pDb, pData, nCurrent, 100, &rc); - nCurrent += 100; - } - - testCksumDatabase(pDb, zCksum); - testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); - - for(iTrans=6; iTrans>2 && rc==0; iTrans--){ - tdb_rollback(pDb, iTrans); - nCurrent -= 100; - testCksumDatabase(pDb, zCksum); - testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); - } - - if( i%2 ){ - tdb_rollback(pDb, 0); - nCurrent -= 100; - testCksumDatabase(pDb, zCksum); - testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); - }else{ - tdb_commit(pDb, 0); - } - } - testCaseFinish(rc); - - skip_rollback_test: - tdb_close(pDb); - testCksumArrayFree(pCksum); - return rc; -} - -void test_rollback( - const char *zSystem, - const char *zPattern, - int *pRc -){ - if( *pRc==0 ){ - int bRun = 1; - - if( zPattern ){ - char *zName = getName(zSystem); - bRun = testGlobMatch(zPattern, zName); - testFree(zName); - } - - if( bRun ){ - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 }; - Datasource *pData = testDatasourceNew(&defn); - *pRc = rollback_test_1(zSystem, pData); - testDatasourceFree(pData); - } - } -} diff --git a/ext/lsm1/lsm-test/lsmtest4.c b/ext/lsm1/lsm-test/lsmtest4.c deleted file mode 100644 index a47241db9..000000000 --- a/ext/lsm1/lsm-test/lsmtest4.c +++ /dev/null @@ -1,127 +0,0 @@ - -/* -** This file contains test cases involving multiple database clients. -*/ - -#include "lsmtest.h" - -/* -** The following code implements test cases "mc1.*". -** -** This test case uses one writer and $nReader readers. All connections -** are driven by a single thread. All connections are opened at the start -** of the test and remain open until the test is finished. -** -** The test consists of $nStep steps. Each step the following is performed: -** -** 1. The writer inserts $nWriteStep records into the db. -** -** 2. The writer checks that the contents of the db are as expected. -** -** 3. Each reader that currently has an open read transaction also checks -** that the contents of the db are as expected (according to the snapshot -** the read transaction is reading - see below). -** -** After step 1, reader 1 opens a read transaction. After step 2, reader -** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1 -** closes the current read transaction and opens a new one. And so on. -** The result is that at step N (for N > $nReader), there exists a reader -** with an open read transaction reading the snapshot committed following -** steps (N-$nReader-1) to N. -*/ -typedef struct Mctest Mctest; -struct Mctest { - DatasourceDefn defn; /* Datasource to use */ - int nStep; /* Total number of steps in test */ - int nWriteStep; /* Number of rows to insert each step */ - int nReader; /* Number of read connections */ -}; -static void do_mc_test( - const char *zSystem, /* Database system to test */ - Mctest *pTest, - int *pRc /* IN/OUT: return code */ -){ - const int nDomain = pTest->nStep * pTest->nWriteStep; - Datasource *pData; /* Source of data */ - TestDb *pDb; /* First database connection (writer) */ - int iReader; /* Used to iterate through aReader */ - int iStep; /* Current step in test */ - int iDot = 0; /* Current step in test */ - - /* Array of reader connections */ - struct Reader { - TestDb *pDb; /* Connection handle */ - int iLast; /* Current snapshot contains keys 0..iLast */ - } *aReader; - - /* Create a data source */ - pData = testDatasourceNew(&pTest->defn); - - /* Open the writer connection */ - pDb = testOpen(zSystem, 1, pRc); - - /* Allocate aReader */ - aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader); - for(iReader=0; iReader<pTest->nReader; iReader++){ - aReader[iReader].pDb = testOpen(zSystem, 0, pRc); - } - - for(iStep=0; iStep<pTest->nStep; iStep++){ - int iLast; - int iBegin; /* Start read trans using aReader[iBegin] */ - - /* Insert nWriteStep more records into the database */ - int iFirst = iStep*pTest->nWriteStep; - testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc); - - /* Check that the db is Ok according to the writer */ - iLast = (iStep+1) * pTest->nWriteStep - 1; - testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc); - - /* Have reader (iStep % nReader) open a read transaction here. */ - iBegin = (iStep % pTest->nReader); - if( iBegin<iStep ) tdb_commit(aReader[iBegin].pDb, 0); - tdb_begin(aReader[iBegin].pDb, 1); - aReader[iBegin].iLast = iLast; - - /* Check that the db is Ok for each open reader */ - for(iReader=0; iReader<pTest->nReader && aReader[iReader].iLast; iReader++){ - iLast = aReader[iReader].iLast; - testDbContents( - aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc - ); - } - - /* Report progress */ - testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot); - } - - /* Close all readers */ - for(iReader=0; iReader<pTest->nReader; iReader++){ - testClose(&aReader[iReader].pDb); - } - testFree(aReader); - - /* Close the writer-connection and free the datasource */ - testClose(&pDb); - testDatasourceFree(pData); -} - - -void test_mc( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - Mctest aTest[] = { - { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 }, - }; - - for(i=0; i<ArraySize(aTest); i++){ - if( testCaseBegin(pRc, zPattern, "mc1.%s.%d", zSystem, i) ){ - do_mc_test(zSystem, &aTest[i], pRc); - testCaseFinish(*pRc); - } - } -} diff --git a/ext/lsm1/lsm-test/lsmtest5.c b/ext/lsm1/lsm-test/lsmtest5.c deleted file mode 100644 index f36184e79..000000000 --- a/ext/lsm1/lsm-test/lsmtest5.c +++ /dev/null @@ -1,633 +0,0 @@ - -/* -** This file is broken into three semi-autonomous parts: -** -** 1. The database functions. -** 2. The thread wrappers. -** 3. The implementation of the mt1.* tests. -*/ - -/************************************************************************* -** DATABASE CONTENTS: -** -** The database contains up to N key/value pairs, where N is some large -** number (say 10,000,000). Keys are integer values between 0 and (N-1). -** The value associated with each key is a pseudo-random blob of data. -** -** Key/value pair keys are encoded as the two bytes "k." followed by a -** 10-digit decimal number. i.e. key 45 -> "k.0000000045". -** -** As well as the key/value pairs, the database also contains checksum -** entries. The checksums form a hierarchy - for every F key/value -** entries there is one level 1 checksum. And for each F level 1 checksums -** there is one level 2 checksum. And so on. -** -** Checksum keys are encoded as the two byte "c." followed by the -** checksum level, followed by a 10 digit decimal number containing -** the value of the first key that contributes to the checksum value. -** For example, assuming F==10, the level 1 checksum that spans keys -** 10 to 19 is "c.1.0000000010". -** -** Clients may perform one of two operations on the database: a read -** or a write. -** -** READ OPERATIONS: -** -** A read operation scans a range of F key/value pairs. It computes -** the expected checksum and then compares the computed value to the -** actual value stored in the level 1 checksum entry. It then scans -** the group of F level 1 checksums, and compares the computed checksum -** to the associated level 2 checksum value, and so on until the -** highest level checksum value has been verified. -** -** If a checksum ever fails to match the expected value, the test -** has failed. -** -** WRITE OPERATIONS: -** -** A write operation involves writing (possibly clobbering) a single -** key/value pair. The associated level 1 checksum is then recalculated -** updated. Then the level 2 checksum, and so on until the highest -** level checksum has been modified. -** -** All updates occur inside a single transaction. -** -** INTERFACE: -** -** The interface used by test cases to read and write the db consists -** of type DbParameters and the following functions: -** -** dbReadOperation() -** dbWriteOperation() -*/ - -#include "lsmtest.h" - -typedef struct DbParameters DbParameters; -struct DbParameters { - int nFanout; /* Checksum fanout (F) */ - int nKey; /* Size of key space (N) */ -}; - -#define DB_KEY_BYTES (2+5+10+1) - -/* -** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. -** This function populates the buffer with a nul-terminated key string -** corresponding to key iKey. -*/ -static void dbFormatKey( - DbParameters *pParam, - int iLevel, - int iKey, /* Key value */ - char *aBuf /* Write key string here */ -){ - if( iLevel==0 ){ - snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey); - }else{ - int f = 1; - int i; - for(i=0; i<iLevel; i++) f = f * pParam->nFanout; - snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f)); - } -} - -/* -** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. -** This function populates the buffer with the string representation of -** checksum value iVal. -*/ -static void dbFormatCksumValue(u32 iVal, char *aBuf){ - snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal); -} - -/* -** Return the highest level of checksum in the database described -** by *pParam. -*/ -static int dbMaxLevel(DbParameters *pParam){ - int iMax; - int n = 1; - for(iMax=0; n<pParam->nKey; iMax++){ - n = n * pParam->nFanout; - } - return iMax; -} - -static void dbCksum( - void *pCtx, /* IN/OUT: Pointer to u32 containing cksum */ - void *pKey, int nKey, /* Database key. Unused. */ - void *pVal, int nVal /* Database value. Checksum this. */ -){ - u8 *aVal = (u8 *)pVal; - u32 *pCksum = (u32 *)pCtx; - u32 cksum = *pCksum; - int i; - - unused_parameter(pKey); - unused_parameter(nKey); - - for(i=0; i<nVal; i++){ - cksum += (cksum<<3) + (int)aVal[i]; - } - - *pCksum = cksum; -} - -/* -** Compute the value of the checksum stored on level iLevel that contains -** data from key iKey by scanning the pParam->nFanout entries at level -** iLevel-1. -*/ -static u32 dbComputeCksum( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - int iLevel, /* Level of checksum to compute */ - int iKey, /* Compute checksum for this key */ - int *pRc /* IN/OUT: Error code */ -){ - u32 cksum = 0; - if( *pRc==0 ){ - int nFirst; - int nLast; - int iFirst = 0; - int iLast = 0; - int i; - int f = 1; - char zFirst[DB_KEY_BYTES]; - char zLast[DB_KEY_BYTES]; - - assert( iLevel>=1 ); - for(i=0; i<iLevel; i++) f = f * pParam->nFanout; - - iFirst = f*(iKey/f); - iLast = iFirst + f - 1; - dbFormatKey(pParam, iLevel-1, iFirst, zFirst); - dbFormatKey(pParam, iLevel-1, iLast, zLast); - nFirst = strlen(zFirst); - nLast = strlen(zLast); - - *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum); - } - - return cksum; -} - -static void dbReadOperation( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - void (*xDelay)(void *), - void *pDelayCtx, - int iKey, /* Key to read */ - int *pRc /* IN/OUT: Error code */ -){ - const int iMax = dbMaxLevel(pParam); - int i; - - if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc); - for(i=1; *pRc==0 && i<=iMax; i++){ - char zCksum[DB_KEY_BYTES]; - char zKey[DB_KEY_BYTES]; - u32 iCksum = 0; - - iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); - if( iCksum ){ - if( xDelay && i==1 ) xDelay(pDelayCtx); - dbFormatCksumValue(iCksum, zCksum); - dbFormatKey(pParam, i, iKey, zKey); - testFetchStr(pDb, zKey, zCksum, pRc); - } - } - if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); -} - -static int dbWriteOperation( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - int iKey, /* Key to write to */ - const char *zValue, /* Nul-terminated value to write */ - int *pRc /* IN/OUT: Error code */ -){ - const int iMax = dbMaxLevel(pParam); - char zKey[DB_KEY_BYTES]; - int i; - int rc; - - assert( iKey>=0 && iKey<pParam->nKey ); - dbFormatKey(pParam, 0, iKey, zKey); - - /* Open a write transaction. This may fail - SQLITE4_BUSY */ - if( *pRc==0 && tdb_transaction_support(pDb) ){ - rc = tdb_begin(pDb, 2); - if( rc==5 ) return 0; - *pRc = rc; - } - - testWriteStr(pDb, zKey, zValue, pRc); - for(i=1; i<=iMax; i++){ - char zCksum[DB_KEY_BYTES]; - u32 iCksum = 0; - - iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); - dbFormatCksumValue(iCksum, zCksum); - dbFormatKey(pParam, i, iKey, zKey); - testWriteStr(pDb, zKey, zCksum, pRc); - } - if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); - return 1; -} - -/************************************************************************* -** The following block contains testXXX() functions that implement a -** wrapper around the systems native multi-thread support. There are no -** synchronization primitives - just functions to launch and join -** threads. Wrapper functions are: -** -** testThreadSupport() -** -** testThreadInit() -** testThreadShutdown() -** testThreadLaunch() -** testThreadWait() -** -** testThreadSetHalt() -** testThreadGetHalt() -** testThreadSetResult() -** testThreadGetResult() -** -** testThreadEnterMutex() -** testThreadLeaveMutex() -*/ -typedef struct ThreadSet ThreadSet; -#ifdef LSM_MUTEX_PTHREADS - -#include <pthread.h> -#include <unistd.h> - -typedef struct Thread Thread; -struct Thread { - int rc; - char *zMsg; - pthread_t id; - void (*xMain)(ThreadSet *, int, void *); - void *pCtx; - ThreadSet *pThreadSet; -}; - -struct ThreadSet { - int bHalt; /* Halt flag */ - int nThread; /* Number of threads */ - Thread *aThread; /* Array of Thread structures */ - pthread_mutex_t mutex; /* Mutex used for cheating */ -}; - -/* -** Return true if this build supports threads, or false otherwise. If -** this function returns false, no other testThreadXXX() functions should -** be called. -*/ -static int testThreadSupport(){ return 1; } - -/* -** Allocate and return a thread-set handle with enough space allocated -** to handle up to nMax threads. Each call to this function should be -** matched by a call to testThreadShutdown() to delete the object. -*/ -static ThreadSet *testThreadInit(int nMax){ - int nByte; /* Total space to allocate */ - ThreadSet *p; /* Return value */ - - nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax; - p = (ThreadSet *)testMalloc(nByte); - p->nThread = nMax; - p->aThread = (Thread *)&p[1]; - pthread_mutex_init(&p->mutex, 0); - - return p; -} - -/* -** Delete a thread-set object and release all resources held by it. -*/ -static void testThreadShutdown(ThreadSet *p){ - int i; - for(i=0; i<p->nThread; i++){ - testFree(p->aThread[i].zMsg); - } - pthread_mutex_destroy(&p->mutex); - testFree(p); -} - -static void *ttMain(void *pArg){ - Thread *pThread = (Thread *)pArg; - int iThread; - iThread = (pThread - pThread->pThreadSet->aThread); - pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx); - return 0; -} - -/* -** Launch a new thread. -*/ -static int testThreadLaunch( - ThreadSet *p, - int iThread, - void (*xMain)(ThreadSet *, int, void *), - void *pCtx -){ - int rc; - Thread *pThread; - - assert( iThread>=0 && iThread<p->nThread ); - - pThread = &p->aThread[iThread]; - assert( pThread->pThreadSet==0 ); - pThread->xMain = xMain; - pThread->pCtx = pCtx; - pThread->pThreadSet = p; - rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread); - - return rc; -} - -/* -** Set the thread-set "halt" flag. -*/ -static void testThreadSetHalt(ThreadSet *pThreadSet){ - pThreadSet->bHalt = 1; -} - -/* -** Return the current value of the thread-set "halt" flag. -*/ -static int testThreadGetHalt(ThreadSet *pThreadSet){ - return pThreadSet->bHalt; -} - -static void testThreadSleep(ThreadSet *pThreadSet, int nMs){ - int nRem = nMs; - while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){ - usleep(50000); - nRem -= 50; - } -} - -/* -** Wait for all threads launched to finish before returning. If nMs -** is greater than zero, set the "halt" flag to tell all threads -** to halt after waiting nMs milliseconds. -*/ -static void testThreadWait(ThreadSet *pThreadSet, int nMs){ - int i; - - testThreadSleep(pThreadSet, nMs); - testThreadSetHalt(pThreadSet); - for(i=0; i<pThreadSet->nThread; i++){ - Thread *pThread = &pThreadSet->aThread[i]; - if( pThread->xMain ){ - pthread_join(pThread->id, 0); - } - } -} - -/* -** Set the result for thread iThread. -*/ -static void testThreadSetResult( - ThreadSet *pThreadSet, /* Thread-set handle */ - int iThread, /* Set result for this thread */ - int rc, /* Result error code */ - char *zFmt, /* Result string format */ - ... /* Result string formatting args... */ -){ - va_list ap; - - testFree(pThreadSet->aThread[iThread].zMsg); - pThreadSet->aThread[iThread].rc = rc; - pThreadSet->aThread[iThread].zMsg = 0; - if( zFmt ){ - va_start(ap, zFmt); - pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap); - va_end(ap); - } -} - -/* -** Retrieve the result for thread iThread. -*/ -static int testThreadGetResult( - ThreadSet *pThreadSet, /* Thread-set handle */ - int iThread, /* Get result for this thread */ - const char **pzRes /* OUT: Pointer to result string */ -){ - if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg; - return pThreadSet->aThread[iThread].rc; -} - -/* -** Enter and leave the test case mutex. -*/ -#if 0 -static void testThreadEnterMutex(ThreadSet *p){ - pthread_mutex_lock(&p->mutex); -} -static void testThreadLeaveMutex(ThreadSet *p){ - pthread_mutex_unlock(&p->mutex); -} -#endif -#endif - -#if !defined(LSM_MUTEX_PTHREADS) -static int testThreadSupport(){ return 0; } - -#define testThreadInit(a) 0 -#define testThreadShutdown(a) -#define testThreadLaunch(a,b,c,d) 0 -#define testThreadWait(a,b) -#define testThreadSetHalt(a) -#define testThreadGetHalt(a) 0 -#define testThreadGetResult(a,b,c) 0 -#define testThreadSleep(a,b) 0 - -static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){ - unused_parameter(a); - unused_parameter(b); - unused_parameter(c); - unused_parameter(d); -} -#endif -/* End of threads wrapper. -*************************************************************************/ - -/************************************************************************* -** Below this point is the third part of this file - the implementation -** of the mt1.* tests. -*/ -typedef struct Mt1Test Mt1Test; -struct Mt1Test { - DbParameters param; /* Description of database to read/write */ - int nReadwrite; /* Number of read/write threads */ - int nFastReader; /* Number of fast reader threads */ - int nSlowReader; /* Number of slow reader threads */ - int nMs; /* How long to run for */ - const char *zSystem; /* Database system to test */ -}; - -typedef struct Mt1DelayCtx Mt1DelayCtx; -struct Mt1DelayCtx { - ThreadSet *pSet; /* Threadset to sleep within */ - int nMs; /* Sleep in ms */ -}; - -static void xMt1Delay(void *pCtx){ - Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx; - testThreadSleep(p->pSet, p->nMs); -} - -#define MT1_THREAD_RDWR 0 -#define MT1_THREAD_SLOW 1 -#define MT1_THREAD_FAST 2 - -static void xMt1Work(lsm_db *pDb, void *pCtx){ -#if 0 - char *z = 0; - lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z); - printf("%s\n", z); - fflush(stdout); -#endif -} - -/* -** This is the main() proc for all threads in test case "mt1". -*/ -static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){ - Mt1Test *p = (Mt1Test *)pCtx; /* Test parameters */ - Mt1DelayCtx delay; - int nRead = 0; /* Number of calls to dbReadOperation() */ - int nWrite = 0; /* Number of completed database writes */ - int rc = 0; /* Error code */ - int iPrng; /* Prng argument variable */ - TestDb *pDb; /* Database handle */ - int eType; - - delay.pSet = pThreadSet; - delay.nMs = 0; - if( iThread<p->nReadwrite ){ - eType = MT1_THREAD_RDWR; - }else if( iThread<(p->nReadwrite+p->nFastReader) ){ - eType = MT1_THREAD_FAST; - }else{ - eType = MT1_THREAD_SLOW; - delay.nMs = (p->nMs / 20); - } - - /* Open a new database connection. Initialize the pseudo-random number - ** argument based on the thread number. */ - iPrng = testPrngValue(iThread); - pDb = testOpen(p->zSystem, 0, &rc); - - if( rc==0 ){ - tdb_lsm_config_work_hook(pDb, xMt1Work, 0); - } - - /* Loop until either an error occurs or some other thread sets the - ** halt flag. */ - while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){ - int iKey; - - /* Perform a read operation on an arbitrarily selected key. */ - iKey = (testPrngValue(iPrng++) % p->param.nKey); - dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc); - if( rc ) continue; - nRead++; - - /* Attempt to write an arbitrary key value pair (and update the associated - ** checksum entries). dbWriteOperation() returns 1 if the write is - ** successful, or 0 if it failed with an LSM_BUSY error. */ - if( eType==MT1_THREAD_RDWR ){ - char aValue[50]; - char aRnd[25]; - - iKey = (testPrngValue(iPrng++) % p->param.nKey); - testPrngString(iPrng, aRnd, sizeof(aRnd)); - iPrng += sizeof(aRnd); - snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd); - nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc); - } - } - testClose(&pDb); - - /* If an error has occured, set the thread error code and the threadset - ** halt flag to tell the other test threads to halt. Otherwise, set the - ** thread error code to 0 and post a message with the number of read - ** and write operations completed. */ - if( rc ){ - testThreadSetResult(pThreadSet, iThread, rc, 0); - testThreadSetHalt(pThreadSet); - }else{ - testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite); - } -} - -static void do_test_mt1( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Mt1Test aTest[] = { - /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */ - { {10, 1000}, 4, 0, 0, 10000, 0 }, - { {10, 1000}, 4, 4, 2, 100000, 0 }, - { {10, 100000}, 4, 0, 0, 10000, 0 }, - { {10, 100000}, 4, 4, 2, 100000, 0 }, - }; - int i; - - for(i=0; *pRc==0 && i<ArraySize(aTest); i++){ - Mt1Test *p = &aTest[i]; - int bRun = testCaseBegin(pRc, zPattern, - "mt1.%s.db=%d,%d.ms=%d.rdwr=%d.fast=%d.slow=%d", - zSystem, p->param.nFanout, p->param.nKey, - p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader - ); - if( bRun ){ - TestDb *pDb; - ThreadSet *pSet; - int iThread; - int nThread; - - p->zSystem = zSystem; - pDb = testOpen(zSystem, 1, pRc); - - nThread = p->nReadwrite + p->nFastReader + p->nSlowReader; - pSet = testThreadInit(nThread); - for(iThread=0; *pRc==0 && iThread<nThread; iThread++){ - testThreadLaunch(pSet, iThread, mt1Main, (void *)p); - } - - testThreadWait(pSet, p->nMs); - for(iThread=0; *pRc==0 && iThread<nThread; iThread++){ - *pRc = testThreadGetResult(pSet, iThread, 0); - } - testCaseFinish(*pRc); - - for(iThread=0; *pRc==0 && iThread<nThread; iThread++){ - const char *zMsg = 0; - *pRc = testThreadGetResult(pSet, iThread, &zMsg); - printf(" Info: thread %d (%d): %s\n", iThread, *pRc, zMsg); - } - - testThreadShutdown(pSet); - testClose(&pDb); - } - } -} - -void test_mt( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - if( testThreadSupport()==0 ) return; - do_test_mt1(zSystem, zPattern, pRc); -} diff --git a/ext/lsm1/lsm-test/lsmtest6.c b/ext/lsm1/lsm-test/lsmtest6.c deleted file mode 100644 index a61b738b8..000000000 --- a/ext/lsm1/lsm-test/lsmtest6.c +++ /dev/null @@ -1,661 +0,0 @@ - -#include "lsmtest.h" - -typedef struct OomTest OomTest; -struct OomTest { - lsm_env *pEnv; - int iNext; /* Next value to pass to testMallocOom() */ - int nFail; /* Number of OOM events injected */ - int bEnable; - int rc; /* Test case error code */ -}; - -static void testOomStart(OomTest *p){ - memset(p, 0, sizeof(OomTest)); - p->iNext = 1; - p->bEnable = 1; - p->nFail = 1; - p->pEnv = tdb_lsm_env(); -} - -static void xOomHook(OomTest *p){ - p->nFail++; -} - -static int testOomContinue(OomTest *p){ - if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){ - return 0; - } - p->nFail = 0; - testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p); - return 1; -} - -static void testOomEnable(OomTest *p, int bEnable){ - p->bEnable = bEnable; - testMallocOomEnable(p->pEnv, bEnable); -} - -static void testOomNext(OomTest *p){ - p->iNext++; -} - -static int testOomHit(OomTest *p){ - return (p->nFail>0); -} - -static int testOomFinish(OomTest *p){ - return p->rc; -} - -static void testOomAssert(OomTest *p, int bVal){ - if( bVal==0 ){ - test_failed(); - p->rc = 1; - } -} - -/* -** Test that the error code matches the state of the OomTest object passed -** as the first argument. Specifically, check that rc is LSM_NOMEM if an -** OOM error has already been injected, or LSM_OK if not. -*/ -static void testOomAssertRc(OomTest *p, int rc){ - testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM); - testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 ); -} - -static void testOomOpen( - OomTest *pOom, - const char *zName, - lsm_db **ppDb, - int *pRc -){ - if( *pRc==LSM_OK ){ - int rc; - rc = lsm_new(tdb_lsm_env(), ppDb); - if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName); - testOomAssertRc(pOom, rc); - *pRc = rc; - } -} - -static void testOomFetch( - OomTest *pOom, - lsm_db *pDb, - void *pKey, int nKey, - void *pVal, int nVal, - int *pRc -){ - testOomAssertRc(pOom, *pRc); - if( *pRc==LSM_OK ){ - lsm_cursor *pCsr; - int rc; - - rc = lsm_csr_open(pDb, &pCsr); - if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0); - testOomAssertRc(pOom, rc); - - if( rc==LSM_OK ){ - const void *p; int n; - testOomAssert(pOom, lsm_csr_valid(pCsr)); - - rc = lsm_csr_key(pCsr, &p, &n); - testOomAssertRc(pOom, rc); - testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) ); - } - - if( rc==LSM_OK ){ - const void *p; int n; - testOomAssert(pOom, lsm_csr_valid(pCsr)); - - rc = lsm_csr_value(pCsr, &p, &n); - testOomAssertRc(pOom, rc); - testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) ); - } - - lsm_csr_close(pCsr); - *pRc = rc; - } -} - -static void testOomWrite( - OomTest *pOom, - lsm_db *pDb, - void *pKey, int nKey, - void *pVal, int nVal, - int *pRc -){ - testOomAssertRc(pOom, *pRc); - if( *pRc==LSM_OK ){ - int rc; - - rc = lsm_insert(pDb, pKey, nKey, pVal, nVal); - testOomAssertRc(pOom, rc); - - *pRc = rc; - } -} - - -static void testOomFetchStr( - OomTest *pOom, - lsm_db *pDb, - const char *zKey, - const char *zVal, - int *pRc -){ - int nKey = strlen(zKey); - int nVal = strlen(zVal); - testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); -} - -static void testOomFetchData( - OomTest *pOom, - lsm_db *pDb, - Datasource *pData, - int iKey, - int *pRc -){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc); -} - -static void testOomWriteStr( - OomTest *pOom, - lsm_db *pDb, - const char *zKey, - const char *zVal, - int *pRc -){ - int nKey = strlen(zKey); - int nVal = strlen(zVal); - testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); -} - -static void testOomWriteData( - OomTest *pOom, - lsm_db *pDb, - Datasource *pData, - int iKey, - int *pRc -){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc); -} - -static void testOomScan( - OomTest *pOom, - lsm_db *pDb, - int bReverse, - const void *pKey, int nKey, - int nScan, - int *pRc -){ - if( *pRc==0 ){ - int rc; - int iScan = 0; - lsm_cursor *pCsr; - int (*xAdvance)(lsm_cursor *) = 0; - - - rc = lsm_csr_open(pDb, &pCsr); - testOomAssertRc(pOom, rc); - - if( rc==LSM_OK ){ - if( bReverse ){ - rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_LE); - xAdvance = lsm_csr_prev; - }else{ - rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_GE); - xAdvance = lsm_csr_next; - } - } - testOomAssertRc(pOom, rc); - - while( rc==LSM_OK && lsm_csr_valid(pCsr) && iScan<nScan ){ - const void *p; int n; - - rc = lsm_csr_key(pCsr, &p, &n); - testOomAssertRc(pOom, rc); - if( rc==LSM_OK ){ - rc = lsm_csr_value(pCsr, &p, &n); - testOomAssertRc(pOom, rc); - } - if( rc==LSM_OK ){ - rc = xAdvance(pCsr); - testOomAssertRc(pOom, rc); - } - iScan++; - } - - lsm_csr_close(pCsr); - *pRc = rc; - } -} - -#define LSMTEST6_TESTDB "testdb.lsm" - -void testDeleteLsmdb(const char *zFile){ - char *zLog = testMallocPrintf("%s-log", zFile); - char *zShm = testMallocPrintf("%s-shm", zFile); - unlink(zFile); - unlink(zLog); - unlink(zShm); - testFree(zLog); - testFree(zShm); -} - -static void copy_file(const char *zFrom, const char *zTo, int isDatabase){ - - if( access(zFrom, F_OK) ){ - unlink(zTo); - }else{ - int fd1; - int fd2; - off_t sz; - off_t i; - struct stat buf; - u8 *aBuf; - - fd1 = open(zFrom, O_RDONLY | _O_BINARY, 0644); - fd2 = open(zTo, O_RDWR | O_CREAT | _O_BINARY, 0644); - - fstat(fd1, &buf); - sz = buf.st_size; - ftruncate(fd2, sz); - - aBuf = testMalloc(4096); - for(i=0; i<sz; i+=4096){ - int bLockPage = isDatabase && i == 0; - int nByte = MIN((bLockPage ? 4066 : 4096), sz - i); - memset(aBuf, 0, 4096); - read(fd1, aBuf, nByte); - write(fd2, aBuf, nByte); - if( bLockPage ){ - lseek(fd1, 4096, SEEK_SET); - lseek(fd2, 4096, SEEK_SET); - } - } - testFree(aBuf); - - close(fd1); - close(fd2); - } -} - -void testCopyLsmdb(const char *zFrom, const char *zTo){ - char *zLog1 = testMallocPrintf("%s-log", zFrom); - char *zLog2 = testMallocPrintf("%s-log", zTo); - char *zShm1 = testMallocPrintf("%s-shm", zFrom); - char *zShm2 = testMallocPrintf("%s-shm", zTo); - - unlink(zShm2); - unlink(zLog2); - unlink(zTo); - copy_file(zFrom, zTo, 1); - copy_file(zLog1, zLog2, 0); - copy_file(zShm1, zShm2, 0); - - testFree(zLog1); testFree(zLog2); testFree(zShm1); testFree(zShm2); -} - -/* -** File zFile is the path to a database. This function makes backups -** of the database file and its log as follows: -** -** cp $(zFile) $(zFile)-save -** cp $(zFile)-$(zAux) $(zFile)-save-$(zAux) -** -** Function testRestoreDb() can be used to copy the files back in the -** other direction. -*/ -void testSaveDb(const char *zFile, const char *zAux){ - char *zLog = testMallocPrintf("%s-%s", zFile, zAux); - char *zFileSave = testMallocPrintf("%s-save", zFile); - char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux); - - unlink(zFileSave); - unlink(zLogSave); - copy_file(zFile, zFileSave, 1); - copy_file(zLog, zLogSave, 0); - - testFree(zLog); testFree(zFileSave); testFree(zLogSave); -} - -/* -** File zFile is the path to a database. This function restores -** a backup of the database made by a previous call to testSaveDb(). -** Specifically, it does the equivalent of: -** -** cp $(zFile)-save $(zFile) -** cp $(zFile)-save-$(zAux) $(zFile)-$(zAux) -*/ -void testRestoreDb(const char *zFile, const char *zAux){ - char *zLog = testMallocPrintf("%s-%s", zFile, zAux); - char *zFileSave = testMallocPrintf("%s-save", zFile); - char *zLogSave = testMallocPrintf("%s-%s-save", zFile, zAux); - - copy_file(zFileSave, zFile, 1); - copy_file(zLogSave, zLog, 0); - - testFree(zLog); testFree(zFileSave); testFree(zLogSave); -} - - -static int lsmWriteStr(lsm_db *pDb, const char *zKey, const char *zVal){ - int nKey = strlen(zKey); - int nVal = strlen(zVal); - return lsm_insert(pDb, (void *)zKey, nKey, (void *)zVal, nVal); -} - -static void setup_delete_db(void){ - testDeleteLsmdb(LSMTEST6_TESTDB); -} - -/* -** Create a small database. With the following content: -** -** "one" -> "one" -** "two" -> "four" -** "three" -> "nine" -** "four" -> "sixteen" -** "five" -> "twentyfive" -** "six" -> "thirtysix" -** "seven" -> "fourtynine" -** "eight" -> "sixtyfour" -*/ -static void setup_populate_db(void){ - const char *azStr[] = { - "one", "one", - "two", "four", - "three", "nine", - "four", "sixteen", - "five", "twentyfive", - "six", "thirtysix", - "seven", "fourtynine", - "eight", "sixtyfour", - }; - int rc; - int ii; - lsm_db *pDb; - - testDeleteLsmdb(LSMTEST6_TESTDB); - - rc = lsm_new(tdb_lsm_env(), &pDb); - if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB); - - for(ii=0; rc==LSM_OK && ii<ArraySize(azStr); ii+=2){ - rc = lsmWriteStr(pDb, azStr[ii], azStr[ii+1]); - } - lsm_close(pDb); - - testSaveDb(LSMTEST6_TESTDB, "log"); - assert( rc==LSM_OK ); -} - -static Datasource *getDatasource(void){ - const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 }; - return testDatasourceNew(&defn); -} - -/* -** Set up a database file with the following properties: -** -** * Page size is 1024 bytes. -** * Block size is 64 KB. -** * Contains 5000 key-value pairs starting at 0 from the -** datasource returned getDatasource(). -*/ -static void setup_populate_db2(void){ - Datasource *pData; - int ii; - int rc; - int nBlocksize = 64*1024; - int nPagesize = 1024; - int nWritebuffer = 4*1024; - lsm_db *pDb; - - testDeleteLsmdb(LSMTEST6_TESTDB); - rc = lsm_new(tdb_lsm_env(), &pDb); - if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB); - - lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &nBlocksize); - lsm_config(pDb, LSM_CONFIG_PAGE_SIZE, &nPagesize); - lsm_config(pDb, LSM_CONFIG_AUTOFLUSH, &nWritebuffer); - - pData = getDatasource(); - for(ii=0; rc==LSM_OK && ii<5000; ii++){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, ii, &pKey, &nKey, &pVal, &nVal); - lsm_insert(pDb, pKey, nKey, pVal, nVal); - } - testDatasourceFree(pData); - lsm_close(pDb); - - testSaveDb(LSMTEST6_TESTDB, "log"); - assert( rc==LSM_OK ); -} - -/* -** Test the results of OOM conditions in lsm_new(). -*/ -static void simple_oom_1(OomTest *pOom){ - int rc; - lsm_db *pDb; - - rc = lsm_new(tdb_lsm_env(), &pDb); - testOomAssertRc(pOom, rc); - - lsm_close(pDb); -} - -/* -** Test the results of OOM conditions in lsm_open(). -*/ -static void simple_oom_2(OomTest *pOom){ - int rc; - lsm_db *pDb; - - rc = lsm_new(tdb_lsm_env(), &pDb); - if( rc==LSM_OK ){ - rc = lsm_open(pDb, "testdb.lsm"); - } - testOomAssertRc(pOom, rc); - - lsm_close(pDb); -} - -/* -** Test the results of OOM conditions in simple fetch operations. -*/ -static void simple_oom_3(OomTest *pOom){ - int rc = LSM_OK; - lsm_db *pDb; - - testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); - - testOomFetchStr(pOom, pDb, "four", "sixteen", &rc); - testOomFetchStr(pOom, pDb, "seven", "fourtynine", &rc); - testOomFetchStr(pOom, pDb, "one", "one", &rc); - testOomFetchStr(pOom, pDb, "eight", "sixtyfour", &rc); - - lsm_close(pDb); -} - -/* -** Test the results of OOM conditions in simple write operations. -*/ -static void simple_oom_4(OomTest *pOom){ - int rc = LSM_OK; - lsm_db *pDb; - - testDeleteLsmdb(LSMTEST6_TESTDB); - testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); - - testOomWriteStr(pOom, pDb, "123", "onetwothree", &rc); - testOomWriteStr(pOom, pDb, "456", "fourfivesix", &rc); - testOomWriteStr(pOom, pDb, "789", "seveneightnine", &rc); - testOomWriteStr(pOom, pDb, "123", "teneleventwelve", &rc); - testOomWriteStr(pOom, pDb, "456", "fourteenfifteensixteen", &rc); - - lsm_close(pDb); -} - -static void simple_oom_5(OomTest *pOom){ - Datasource *pData = getDatasource(); - int rc = LSM_OK; - lsm_db *pDb; - - testRestoreDb(LSMTEST6_TESTDB, "log"); - testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); - - testOomFetchData(pOom, pDb, pData, 3333, &rc); - testOomFetchData(pOom, pDb, pData, 0, &rc); - testOomFetchData(pOom, pDb, pData, 4999, &rc); - - lsm_close(pDb); - testDatasourceFree(pData); -} - -static void simple_oom_6(OomTest *pOom){ - Datasource *pData = getDatasource(); - int rc = LSM_OK; - lsm_db *pDb; - - testRestoreDb(LSMTEST6_TESTDB, "log"); - testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); - - testOomWriteData(pOom, pDb, pData, 5000, &rc); - testOomWriteData(pOom, pDb, pData, 5001, &rc); - testOomWriteData(pOom, pDb, pData, 5002, &rc); - testOomFetchData(pOom, pDb, pData, 5001, &rc); - testOomFetchData(pOom, pDb, pData, 1234, &rc); - - lsm_close(pDb); - testDatasourceFree(pData); -} - -static void simple_oom_7(OomTest *pOom){ - Datasource *pData = getDatasource(); - int rc = LSM_OK; - lsm_db *pDb; - - testRestoreDb(LSMTEST6_TESTDB, "log"); - testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); - testOomScan(pOom, pDb, 0, "abc", 3, 20, &rc); - lsm_close(pDb); - testDatasourceFree(pData); -} - -static void simple_oom_8(OomTest *pOom){ - Datasource *pData = getDatasource(); - int rc = LSM_OK; - lsm_db *pDb; - testRestoreDb(LSMTEST6_TESTDB, "log"); - testOomOpen(pOom, LSMTEST6_TESTDB, &pDb, &rc); - testOomScan(pOom, pDb, 1, "xyz", 3, 20, &rc); - lsm_close(pDb); - testDatasourceFree(pData); -} - -/* -** This test case has two clients connected to a database. The first client -** hits an OOM while writing to the database. Check that the second -** connection is still able to query the db following the OOM. -*/ -static void simple_oom2_1(OomTest *pOom){ - const int nRecord = 100; /* Number of records initially in db */ - const int nIns = 10; /* Number of records inserted with OOM */ - - Datasource *pData = getDatasource(); - int rc = LSM_OK; - lsm_db *pDb1; - lsm_db *pDb2; - int i; - - testDeleteLsmdb(LSMTEST6_TESTDB); - - /* Open the two connections. Initialize the in-memory tree so that it - ** contains 100 records. Do all this with OOM injection disabled. */ - testOomEnable(pOom, 0); - testOomOpen(pOom, LSMTEST6_TESTDB, &pDb1, &rc); - testOomOpen(pOom, LSMTEST6_TESTDB, &pDb2, &rc); - for(i=0; i<nRecord; i++){ - testOomWriteData(pOom, pDb1, pData, i, &rc); - } - testOomEnable(pOom, 1); - assert( rc==0 ); - - /* Insert 10 more records using pDb1. Stop when an OOM is encountered. */ - for(i=nRecord; i<nRecord+nIns; i++){ - testOomWriteData(pOom, pDb1, pData, i, &rc); - if( rc ) break; - } - testOomAssertRc(pOom, rc); - - /* Switch off OOM injection. Write a few rows using pDb2. Then check - ** that the database may be successfully queried. */ - testOomEnable(pOom, 0); - rc = 0; - for(; i<nRecord+nIns && rc==0; i++){ - testOomWriteData(pOom, pDb2, pData, i, &rc); - } - for(i=0; i<nRecord+nIns; i++) testOomFetchData(pOom, pDb2, pData, i, &rc); - testOomEnable(pOom, 1); - - lsm_close(pDb1); - lsm_close(pDb2); - testDatasourceFree(pData); -} - - -static void do_test_oom1(const char *zPattern, int *pRc){ - struct SimpleOom { - const char *zName; - void (*xSetup)(void); - void (*xFunc)(OomTest *); - } aSimple[] = { - { "oom1.lsm.1", setup_delete_db, simple_oom_1 }, - { "oom1.lsm.2", setup_delete_db, simple_oom_2 }, - { "oom1.lsm.3", setup_populate_db, simple_oom_3 }, - { "oom1.lsm.4", setup_delete_db, simple_oom_4 }, - { "oom1.lsm.5", setup_populate_db2, simple_oom_5 }, - { "oom1.lsm.6", setup_populate_db2, simple_oom_6 }, - { "oom1.lsm.7", setup_populate_db2, simple_oom_7 }, - { "oom1.lsm.8", setup_populate_db2, simple_oom_8 }, - - { "oom2.lsm.1", setup_delete_db, simple_oom2_1 }, - }; - int i; - - for(i=0; i<ArraySize(aSimple); i++){ - if( *pRc==0 && testCaseBegin(pRc, zPattern, "%s", aSimple[i].zName) ){ - OomTest t; - - if( aSimple[i].xSetup ){ - aSimple[i].xSetup(); - } - - for(testOomStart(&t); testOomContinue(&t); testOomNext(&t)){ - aSimple[i].xFunc(&t); - } - - printf("(%d injections).", t.iNext-2); - testCaseFinish( (*pRc = testOomFinish(&t)) ); - testMallocOom(tdb_lsm_env(), 0, 0, 0, 0); - } - } -} - -void test_oom( - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - do_test_oom1(zPattern, pRc); -} diff --git a/ext/lsm1/lsm-test/lsmtest7.c b/ext/lsm1/lsm-test/lsmtest7.c deleted file mode 100644 index 2d26b53a7..000000000 --- a/ext/lsm1/lsm-test/lsmtest7.c +++ /dev/null @@ -1,206 +0,0 @@ - - -#include "lsmtest.h" - - -/* -** Test that the rules for when lsm_csr_next() and lsm_csr_prev() are -** enforced. Specifically: -** -** * Both functions always return LSM_MISUSE if the cursor is at EOF -** when they are called. -** -** * lsm_csr_next() may only be used after lsm_csr_seek(LSM_SEEK_GE) or -** lsm_csr_first(). -** -** * lsm_csr_prev() may only be used after lsm_csr_seek(LSM_SEEK_LE) or -** lsm_csr_last(). -*/ -static void do_test_api1_lsm(lsm_db *pDb, int *pRc){ - int ret; - lsm_cursor *pCsr; - lsm_cursor *pCsr2; - int nKey; - const void *pKey; - - ret = lsm_csr_open(pDb, &pCsr); - testCompareInt(LSM_OK, ret, pRc); - - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - - ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_GE); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - - ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LE); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_OK, ret, pRc); - - ret = lsm_csr_seek(pCsr, "jjj", 3, LSM_SEEK_LEFAST); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - - ret = lsm_csr_key(pCsr, &pKey, &nKey); - testCompareInt(LSM_OK, ret, pRc); - - ret = lsm_csr_open(pDb, &pCsr2); - testCompareInt(LSM_OK, ret, pRc); - - ret = lsm_csr_seek(pCsr2, pKey, nKey, LSM_SEEK_EQ); - testCompareInt(LSM_OK, ret, pRc); - testCompareInt(1, lsm_csr_valid(pCsr2), pRc); - ret = lsm_csr_next(pCsr2); - testCompareInt(LSM_MISUSE, ret, pRc); - ret = lsm_csr_prev(pCsr2); - testCompareInt(LSM_MISUSE, ret, pRc); - - lsm_csr_close(pCsr2); - - ret = lsm_csr_first(pCsr); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - - ret = lsm_csr_last(pCsr); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - - ret = lsm_csr_first(pCsr); - while( lsm_csr_valid(pCsr) ){ - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_OK, ret, pRc); - } - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - - ret = lsm_csr_last(pCsr); - while( lsm_csr_valid(pCsr) ){ - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_OK, ret, pRc); - } - ret = lsm_csr_prev(pCsr); - testCompareInt(LSM_OK, ret, pRc); - ret = lsm_csr_next(pCsr); - testCompareInt(LSM_MISUSE, ret, pRc); - - lsm_csr_close(pCsr); -} - -static void do_test_api1(const char *zPattern, int *pRc){ - if( testCaseBegin(pRc, zPattern, "api1.lsm") ){ - const DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 200, 250 }; - Datasource *pData; - TestDb *pDb; - int rc = 0; - - pDb = testOpen("lsm_lomem", 1, &rc); - pData = testDatasourceNew(&defn); - testWriteDatasourceRange(pDb, pData, 0, 1000, pRc); - - do_test_api1_lsm(tdb_lsm(pDb), pRc); - - testDatasourceFree(pData); - testClose(&pDb); - - testCaseFinish(*pRc); - } -} - -static lsm_db *newLsmConnection( - const char *zDb, - int nPgsz, - int nBlksz, - int *pRc -){ - lsm_db *db = 0; - if( *pRc==0 ){ - int n1 = nPgsz; - int n2 = nBlksz; - *pRc = lsm_new(tdb_lsm_env(), &db); - if( *pRc==0 ){ - if( n1 ) lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1); - if( n2 ) lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2); - *pRc = lsm_open(db, "testdb.lsm"); - } - } - return db; -} - -static void testPagesize(lsm_db *db, int nPgsz, int nBlksz, int *pRc){ - if( *pRc==0 ){ - int n1 = 0; - int n2 = 0; - - lsm_config(db, LSM_CONFIG_PAGE_SIZE, &n1); - lsm_config(db, LSM_CONFIG_BLOCK_SIZE, &n2); - - testCompareInt(n1, nPgsz, pRc); - testCompareInt(n2, nBlksz, pRc); - } -} - -/* -** Test case "api2" tests that the default page and block sizes of a -** database may only be modified before lsm_open() is called. And that -** after lsm_open() is called lsm_config() may be used to read the -** actual page and block size of the db. -*/ -static void do_test_api2(const char *zPattern, int *pRc){ - if( *pRc==0 && testCaseBegin(pRc, zPattern, "api2.lsm") ){ - lsm_db *db1 = 0; - lsm_db *db2 = 0; - - testDeleteLsmdb("testdb.lsm"); - db1 = newLsmConnection("testdb.lsm", 0, 0, pRc); - testPagesize(db1, 4096, 1024, pRc); - db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc); - testPagesize(db2, 4096, 1024, pRc); - lsm_close(db1); - lsm_close(db2); - - testDeleteLsmdb("testdb.lsm"); - db1 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc); - testPagesize(db1, 1024, 64*1024, pRc); - db2 = newLsmConnection("testdb.lsm", 0, 0, pRc); - testPagesize(db2, 1024, 64*1024, pRc); - lsm_close(db1); - lsm_close(db2); - - testDeleteLsmdb("testdb.lsm"); - db1 = newLsmConnection("testdb.lsm", 8192, 2*1024, pRc); - testPagesize(db1, 8192, 2*1024, pRc); - db2 = newLsmConnection("testdb.lsm", 1024, 64*1024, pRc); - testPagesize(db2, 8192, 2*1024, pRc); - lsm_close(db1); - lsm_close(db2); - - testCaseFinish(*pRc); - } -} - -void test_api( - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - do_test_api1(zPattern, pRc); - do_test_api2(zPattern, pRc); -} diff --git a/ext/lsm1/lsm-test/lsmtest8.c b/ext/lsm1/lsm-test/lsmtest8.c deleted file mode 100644 index 7efa0dfa6..000000000 --- a/ext/lsm1/lsm-test/lsmtest8.c +++ /dev/null @@ -1,324 +0,0 @@ - -/* -** This file contains test cases to verify that "live-recovery" following -** a mid-transaction failure of a writer process. -*/ - - -/* -** This test file includes lsmInt.h to get access to the definition of the -** ShmHeader structure. This is required to cause strategic damage to the -** shared memory header as part of recovery testing. -*/ -#include "lsmInt.h" - -#include "lsmtest.h" - -typedef struct SetupStep SetupStep; -struct SetupStep { - int bFlush; /* Flush to disk and checkpoint */ - int iInsStart; /* First key-value from ds to insert */ - int nIns; /* Number of rows to insert */ - int iDelStart; /* First key from ds to delete */ - int nDel; /* Number of rows to delete */ -}; - -static void doSetupStep( - TestDb *pDb, - Datasource *pData, - const SetupStep *pStep, - int *pRc -){ - testWriteDatasourceRange(pDb, pData, pStep->iInsStart, pStep->nIns, pRc); - testDeleteDatasourceRange(pDb, pData, pStep->iDelStart, pStep->nDel, pRc); - if( *pRc==0 ){ - int nSave = -1; - int nBuf = 64; - lsm_db *db = tdb_lsm(pDb); - - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); - lsm_begin(db, 1); - lsm_commit(db, 0); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); - - *pRc = lsm_work(db, 0, 0, 0); - if( *pRc==0 ){ - *pRc = lsm_checkpoint(db, 0); - } - } -} - -static void doSetupStepArray( - TestDb *pDb, - Datasource *pData, - const SetupStep *aStep, - int nStep -){ - int i; - for(i=0; i<nStep; i++){ - int rc = 0; - doSetupStep(pDb, pData, &aStep[i], &rc); - assert( rc==0 ); - } -} - -static void setupDatabase1(TestDb *pDb, Datasource **ppData){ - const SetupStep aStep[] = { - { 0, 1, 2000, 0, 0 }, - { 1, 0, 0, 0, 0 }, - { 0, 10001, 1000, 0, 0 }, - }; - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 100, 500}; - Datasource *pData; - - pData = testDatasourceNew(&defn); - doSetupStepArray(pDb, pData, aStep, ArraySize(aStep)); - if( ppData ){ - *ppData = pData; - }else{ - testDatasourceFree(pData); - } -} - -#include <stdio.h> -void testReadFile(const char *zFile, int iOff, void *pOut, int nByte, int *pRc){ - if( *pRc==0 ){ - FILE *fd; - fd = fopen(zFile, "rb"); - if( fd==0 ){ - *pRc = 1; - }else{ - if( 0!=fseek(fd, iOff, SEEK_SET) ){ - *pRc = 1; - }else{ - assert( nByte>=0 ); - if( (size_t)nByte!=fread(pOut, 1, nByte, fd) ){ - *pRc = 1; - } - } - fclose(fd); - } - } -} - -void testWriteFile( - const char *zFile, - int iOff, - void *pOut, - int nByte, - int *pRc -){ - if( *pRc==0 ){ - FILE *fd; - fd = fopen(zFile, "r+b"); - if( fd==0 ){ - *pRc = 1; - }else{ - if( 0!=fseek(fd, iOff, SEEK_SET) ){ - *pRc = 1; - }else{ - assert( nByte>=0 ); - if( (size_t)nByte!=fwrite(pOut, 1, nByte, fd) ){ - *pRc = 1; - } - } - fclose(fd); - } - } -} - -static ShmHeader *getShmHeader(const char *zDb){ - int rc = 0; - char *zShm = testMallocPrintf("%s-shm", zDb); - ShmHeader *pHdr; - - pHdr = testMalloc(sizeof(ShmHeader)); - testReadFile(zShm, 0, (void *)pHdr, sizeof(ShmHeader), &rc); - assert( rc==0 ); - - return pHdr; -} - -/* -** This function makes a copy of the three files associated with LSM -** database zDb (i.e. if zDb is "test.db", it makes copies of "test.db", -** "test.db-log" and "test.db-shm"). -** -** It then opens a new database connection to the copy with the xLock() call -** instrumented so that it appears that some other process already connected -** to the db (holding a shared lock on DMS2). This prevents recovery from -** running. Then: -** -** 1) Check that the checksum of the database is zCksum. -** 2) Write a few keys to the database. Then delete the same keys. -** 3) Check that the checksum is zCksum. -** 4) Flush the db to disk and run a checkpoint. -** 5) Check once more that the checksum is still zCksum. -*/ -static void doLiveRecovery(const char *zDb, const char *zCksum, int *pRc){ - if( *pRc==LSM_OK ){ - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 20, 25, 100, 500}; - Datasource *pData; - const char *zCopy = "testcopy.lsm"; - char zCksum2[TEST_CKSUM_BYTES]; - TestDb *pDb = 0; - int rc; - - pData = testDatasourceNew(&defn); - - testCopyLsmdb(zDb, zCopy); - rc = tdb_lsm_open("test_no_recovery=1", zCopy, 0, &pDb); - if( rc==0 ){ - ShmHeader *pHdr; - lsm_db *db; - testCksumDatabase(pDb, zCksum2); - testCompareStr(zCksum, zCksum2, &rc); - - testWriteDatasourceRange(pDb, pData, 1, 10, &rc); - testDeleteDatasourceRange(pDb, pData, 1, 10, &rc); - - /* Test that the two tree-headers are now consistent. */ - pHdr = getShmHeader(zCopy); - if( rc==0 && memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(pHdr->hdr1)) ){ - rc = 1; - } - testFree(pHdr); - - if( rc==0 ){ - int nBuf = 64; - db = tdb_lsm(pDb); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); - lsm_begin(db, 1); - lsm_commit(db, 0); - rc = lsm_work(db, 0, 0, 0); - } - - testCksumDatabase(pDb, zCksum2); - testCompareStr(zCksum, zCksum2, &rc); - } - - testDatasourceFree(pData); - testClose(&pDb); - testDeleteLsmdb(zCopy); - *pRc = rc; - } -} - -static void doWriterCrash1(int *pRc){ - const int nWrite = 2000; - const int nStep = 10; - const int iWriteStart = 20000; - int rc = 0; - TestDb *pDb = 0; - Datasource *pData = 0; - - rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb); - if( rc==0 ){ - int iDot = 0; - char zCksum[TEST_CKSUM_BYTES]; - int i; - setupDatabase1(pDb, &pData); - testCksumDatabase(pDb, zCksum); - testBegin(pDb, 2, &rc); - for(i=0; rc==0 && i<nWrite; i+=nStep){ - testCaseProgress(i, nWrite, testCaseNDot(), &iDot); - testWriteDatasourceRange(pDb, pData, iWriteStart+i, nStep, &rc); - doLiveRecovery("testdb.lsm", zCksum, &rc); - } - } - testCommit(pDb, 0, &rc); - testClose(&pDb); - testDatasourceFree(pData); - *pRc = rc; -} - -/* -** This test case verifies that inconsistent tree-headers in shared-memory -** are resolved correctly. -*/ -static void doWriterCrash2(int *pRc){ - int rc = 0; - TestDb *pDb = 0; - Datasource *pData = 0; - - rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb); - if( rc==0 ){ - ShmHeader *pHdr1; - ShmHeader *pHdr2; - char zCksum1[TEST_CKSUM_BYTES]; - char zCksum2[TEST_CKSUM_BYTES]; - - pHdr1 = testMalloc(sizeof(ShmHeader)); - pHdr2 = testMalloc(sizeof(ShmHeader)); - setupDatabase1(pDb, &pData); - - /* Grab a copy of the shared-memory header. And the db checksum */ - testReadFile("testdb.lsm-shm", 0, (void *)pHdr1, sizeof(ShmHeader), &rc); - testCksumDatabase(pDb, zCksum1); - - /* Modify the database */ - testBegin(pDb, 2, &rc); - testWriteDatasourceRange(pDb, pData, 30000, 200, &rc); - testCommit(pDb, 0, &rc); - - /* Grab a second copy of the shared-memory header. And the db checksum */ - testReadFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - testCksumDatabase(pDb, zCksum2); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - /* If both tree-headers are valid, tree-header-1 is used. */ - memcpy(&pHdr2->hdr1, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum1, &rc); - - /* If both tree-headers are valid, tree-header-1 is used. */ - memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); - memcpy(&pHdr2->hdr2, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - /* If tree-header 1 is invalid, tree-header-2 is used */ - memcpy(&pHdr2->hdr2, &pHdr2->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->hdr1.aCksum[0] = 5; - pHdr2->hdr1.aCksum[0] = 6; - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - /* If tree-header 2 is invalid, tree-header-1 is used */ - memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); - pHdr2->hdr2.aCksum[0] = 5; - pHdr2->hdr2.aCksum[0] = 6; - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - testFree(pHdr1); - testFree(pHdr2); - testClose(&pDb); - } - - *pRc = rc; -} - -void do_writer_crash_test(const char *zPattern, int *pRc){ - struct Test { - const char *zName; - void (*xFunc)(int *); - } aTest[] = { - { "writercrash1.lsm", doWriterCrash1 }, - { "writercrash2.lsm", doWriterCrash2 }, - }; - int i; - for(i=0; i<ArraySize(aTest); i++){ - struct Test *p = &aTest[i]; - if( testCaseBegin(pRc, zPattern, p->zName) ){ - p->xFunc(pRc); - testCaseFinish(*pRc); - } - } - -} diff --git a/ext/lsm1/lsm-test/lsmtest9.c b/ext/lsm1/lsm-test/lsmtest9.c deleted file mode 100644 index b01de0d4e..000000000 --- a/ext/lsm1/lsm-test/lsmtest9.c +++ /dev/null @@ -1,140 +0,0 @@ - -#include "lsmtest.h" - -#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE -#define DATA_RANDOM TEST_DATASOURCE_RANDOM - -typedef struct Datatest4 Datatest4; - -/* -** Test overview: -** -** 1. Insert (Datatest4.nRec) records into a database. -** -** 2. Repeat (Datatest4.nRepeat) times: -** -** 2a. Delete 2/3 of the records in the database. -** -** 2b. Run lsm_work(nMerge=1). -** -** 2c. Insert as many records as were deleted in 2a. -** -** 2d. Check database content is as expected. -** -** 2e. If (Datatest4.bReopen) is true, close and reopen the database. -*/ -struct Datatest4 { - /* Datasource definition */ - DatasourceDefn defn; - - int nRec; - int nRepeat; - int bReopen; -}; - -static void doDataTest4( - const char *zSystem, /* Database system to test */ - Datatest4 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - lsm_db *db = 0; - TestDb *pDb; - TestDb *pControl; - Datasource *pData; - int i; - int rc = 0; - int iDot = 0; - int bMultiThreaded = 0; /* True for MT LSM database */ - - int nRecOn3 = (p->nRec / 3); - int iData = 0; - - /* Start the test case, open a database and allocate the datasource. */ - rc = testControlDb(&pControl); - pDb = testOpen(zSystem, 1, &rc); - pData = testDatasourceNew(&p->defn); - if( rc==0 ){ - db = tdb_lsm(pDb); - bMultiThreaded = tdb_lsm_multithread(pDb); - } - - testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc); - testWriteDatasourceRange(pDb, pData, iData, nRecOn3*3, &rc); - - for(i=0; rc==0 && i<p->nRepeat; i++){ - - testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc); - testDeleteDatasourceRange(pDb, pData, iData, nRecOn3*2, &rc); - - if( db ){ - int nDone; -#if 0 - fprintf(stderr, "lsm_work() start...\n"); fflush(stderr); -#endif - do { - nDone = 0; - rc = lsm_work(db, 1, (1<<30), &nDone); - }while( rc==0 && nDone>0 ); - if( bMultiThreaded && rc==LSM_BUSY ) rc = LSM_OK; -#if 0 - fprintf(stderr, "lsm_work() done...\n"); fflush(stderr); -#endif - } - -if( i+1<p->nRepeat ){ - iData += (nRecOn3*2); - testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc); - testWriteDatasourceRange(pDb, pData, iData+nRecOn3, nRecOn3*2, &rc); - - testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc); - - /* If Datatest4.bReopen is true, close and reopen the database */ - if( p->bReopen ){ - testReopen(&pDb, &rc); - if( rc==0 ) db = tdb_lsm(pDb); - } -} - - /* Update the progress dots... */ - testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot); - } - - testClose(&pDb); - testClose(&pControl); - testDatasourceFree(pData); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName4(const char *zSystem, Datatest4 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", - zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen - ); - testFree(zData); - return zRet; -} - -void test_data_4( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest4 aTest[] = { - /* defn, nRec, nRepeat, bReopen */ - { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 0 }, - { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 1 }, - }; - - int i; - - for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){ - char *zName = getName4(zSystem, &aTest[i]); - if( testCaseBegin(pRc, zPattern, "%s", zName) ){ - doDataTest4(zSystem, &aTest[i], pRc); - } - testFree(zName); - } -} diff --git a/ext/lsm1/lsm-test/lsmtest_bt.c b/ext/lsm1/lsm-test/lsmtest_bt.c deleted file mode 100644 index 8a4f54a8c..000000000 --- a/ext/lsm1/lsm-test/lsmtest_bt.c +++ /dev/null @@ -1,71 +0,0 @@ - -#include "lsmtest.h" -#include "bt.h" - -int do_bt(int nArg, char **azArg){ - struct Option { - const char *zName; - int bPgno; - int eOpt; - } aOpt [] = { - { "dbhdr", 0, BT_INFO_HDRDUMP }, - { "filename", 0, BT_INFO_FILENAME }, - { "block_freelist", 0, BT_INFO_BLOCK_FREELIST }, - { "page_freelist", 0, BT_INFO_PAGE_FREELIST }, - { "filename", 0, BT_INFO_FILENAME }, - { "page", 1, BT_INFO_PAGEDUMP }, - { "page_ascii", 1, BT_INFO_PAGEDUMP_ASCII }, - { "leaks", 0, BT_INFO_PAGE_LEAKS }, - { 0, 0 } - }; - int iOpt; - int rc; - bt_info buf; - char *zOpt; - char *zFile; - - bt_db *db = 0; - - if( nArg<2 ){ - testPrintUsage("FILENAME OPTION ..."); - return -1; - } - zFile = azArg[0]; - zOpt = azArg[1]; - - rc = testArgSelect(aOpt, "option", zOpt, &iOpt); - if( rc!=0 ) return rc; - if( nArg!=2+aOpt[iOpt].bPgno ){ - testPrintFUsage("FILENAME %s %s", zOpt, aOpt[iOpt].bPgno ? "PGNO" : ""); - return -4; - } - - rc = sqlite4BtNew(sqlite4_env_default(), 0, &db); - if( rc!=SQLITE4_OK ){ - testPrintError("sqlite4BtNew() failed: %d", rc); - return -2; - } - rc = sqlite4BtOpen(db, zFile); - if( rc!=SQLITE4_OK ){ - testPrintError("sqlite4BtOpen() failed: %d", rc); - return -3; - } - - buf.eType = aOpt[iOpt].eOpt; - buf.pgno = 0; - sqlite4_buffer_init(&buf.output, 0); - - if( aOpt[iOpt].bPgno ){ - buf.pgno = (u32)atoi(azArg[2]); - } - - rc = sqlite4BtControl(db, BT_CONTROL_INFO, &buf); - if( rc!=SQLITE4_OK ){ - testPrintError("sqlite4BtControl() failed: %d\n", rc); - return -4; - } - - printf("%s\n", (char*)buf.output.p); - sqlite4_buffer_clear(&buf.output); - return 0; -} diff --git a/ext/lsm1/lsm-test/lsmtest_datasource.c b/ext/lsm1/lsm-test/lsmtest_datasource.c deleted file mode 100644 index 0b0fd94e8..000000000 --- a/ext/lsm1/lsm-test/lsmtest_datasource.c +++ /dev/null @@ -1,96 +0,0 @@ - - -#include "lsmtest.h" - -struct Datasource { - int eType; - - int nMinKey; - int nMaxKey; - int nMinVal; - int nMaxVal; - - char *aKey; - char *aVal; -}; - -void testDatasourceEntry( - Datasource *p, - int iData, - void **ppKey, int *pnKey, - void **ppVal, int *pnVal -){ - assert( (ppKey==0)==(pnKey==0) ); - assert( (ppVal==0)==(pnVal==0) ); - - if( ppKey ){ - int nKey = 0; - switch( p->eType ){ - case TEST_DATASOURCE_RANDOM: { - int nRange = (1 + p->nMaxKey - p->nMinKey); - nKey = (int)( testPrngValue((u32)iData) % nRange ) + p->nMinKey; - testPrngString((u32)iData, p->aKey, nKey); - break; - } - case TEST_DATASOURCE_SEQUENCE: - nKey = sprintf(p->aKey, "%012d", iData); - break; - } - *ppKey = p->aKey; - *pnKey = nKey; - } - if( ppVal ){ - u32 nVal = testPrngValue((u32)iData)%(1+p->nMaxVal-p->nMinVal)+p->nMinVal; - testPrngString((u32)~iData, p->aVal, (int)nVal); - *ppVal = p->aVal; - *pnVal = (int)nVal; - } -} - -void testDatasourceFree(Datasource *p){ - testFree(p); -} - -/* -** Return a pointer to a nul-terminated string that corresponds to the -** contents of the datasource-definition passed as the first argument. -** The caller should eventually free the returned pointer using testFree(). -*/ -char *testDatasourceName(const DatasourceDefn *p){ - char *zRet; - zRet = testMallocPrintf("%s.(%d-%d).(%d-%d)", - (p->eType==TEST_DATASOURCE_SEQUENCE ? "seq" : "rnd"), - p->nMinKey, p->nMaxKey, - p->nMinVal, p->nMaxVal - ); - return zRet; -} - -Datasource *testDatasourceNew(const DatasourceDefn *pDefn){ - Datasource *p; - int nMinKey; - int nMaxKey; - int nMinVal; - int nMaxVal; - - if( pDefn->eType==TEST_DATASOURCE_SEQUENCE ){ - nMinKey = 128; - nMaxKey = 128; - }else{ - nMinKey = MAX(0, pDefn->nMinKey); - nMaxKey = MAX(nMinKey, pDefn->nMaxKey); - } - nMinVal = MAX(0, pDefn->nMinVal); - nMaxVal = MAX(nMinVal, pDefn->nMaxVal); - - p = (Datasource *)testMalloc(sizeof(Datasource) + nMaxKey + nMaxVal + 1); - p->eType = pDefn->eType; - p->nMinKey = nMinKey; - p->nMinVal = nMinVal; - p->nMaxKey = nMaxKey; - p->nMaxVal = nMaxVal; - - p->aKey = (char *)&p[1]; - p->aVal = &p->aKey[nMaxKey]; - return p; -}; diff --git a/ext/lsm1/lsm-test/lsmtest_func.c b/ext/lsm1/lsm-test/lsmtest_func.c deleted file mode 100644 index eb8346aa8..000000000 --- a/ext/lsm1/lsm-test/lsmtest_func.c +++ /dev/null @@ -1,177 +0,0 @@ - -#include "lsmtest.h" - - -int do_work(int nArg, char **azArg){ - struct Option { - const char *zName; - } aOpt [] = { - { "-nmerge" }, - { "-nkb" }, - { 0 } - }; - - lsm_db *pDb; - int rc; - int i; - const char *zDb; - int nMerge = 1; - int nKB = (1<<30); - - if( nArg==0 ) goto usage; - zDb = azArg[nArg-1]; - for(i=0; i<(nArg-1); i++){ - int iSel; - rc = testArgSelect(aOpt, "option", azArg[i], &iSel); - if( rc ) return rc; - switch( iSel ){ - case 0: - i++; - if( i==(nArg-1) ) goto usage; - nMerge = atoi(azArg[i]); - break; - case 1: - i++; - if( i==(nArg-1) ) goto usage; - nKB = atoi(azArg[i]); - break; - } - } - - rc = lsm_new(0, &pDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - }else{ - rc = lsm_open(pDb, zDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - }else{ - int n = -1; - lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n); - n = n*2; - lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n); - - rc = lsm_work(pDb, nMerge, nKB, 0); - if( rc!=LSM_OK ){ - testPrintError("lsm_work(): rc=%d\n", rc); - } - } - } - if( rc==LSM_OK ){ - rc = lsm_checkpoint(pDb, 0); - } - - lsm_close(pDb); - return rc; - - usage: - testPrintUsage("?-optimize? ?-n N? DATABASE"); - return -1; -} - - -/* -** lsmtest show ?-config LSM-CONFIG? DATABASE ?COMMAND ?PGNO?? -*/ -int do_show(int nArg, char **azArg){ - lsm_db *pDb; - int rc; - const char *zDb; - - int eOpt = LSM_INFO_DB_STRUCTURE; - unsigned int iPg = 0; - int bConfig = 0; - const char *zConfig = ""; - - struct Option { - const char *zName; - int bConfig; - int eOpt; - } aOpt [] = { - { "array", 0, LSM_INFO_ARRAY_STRUCTURE }, - { "array-pages", 0, LSM_INFO_ARRAY_PAGES }, - { "blocksize", 1, LSM_CONFIG_BLOCK_SIZE }, - { "pagesize", 1, LSM_CONFIG_PAGE_SIZE }, - { "freelist", 0, LSM_INFO_FREELIST }, - { "page-ascii", 0, LSM_INFO_PAGE_ASCII_DUMP }, - { "page-hex", 0, LSM_INFO_PAGE_HEX_DUMP }, - { 0, 0 } - }; - - char *z = 0; - int iDb = 0; /* Index of DATABASE in azArg[] */ - - /* Check if there is a "-config" option: */ - if( nArg>2 && strlen(azArg[0])>1 - && memcmp(azArg[0], "-config", strlen(azArg[0]))==0 - ){ - zConfig = azArg[1]; - iDb = 2; - } - if( nArg<(iDb+1) ) goto usage; - - if( nArg>(iDb+1) ){ - rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt); - if( rc!=0 ) return rc; - bConfig = aOpt[eOpt].bConfig; - eOpt = aOpt[eOpt].eOpt; - if( (bConfig==0 && eOpt==LSM_INFO_FREELIST) - || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE) - || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE) - ){ - if( nArg!=(iDb+2) ) goto usage; - }else{ - if( nArg!=(iDb+3) ) goto usage; - iPg = atoi(azArg[iDb+2]); - } - } - zDb = azArg[iDb]; - - rc = lsm_new(0, &pDb); - tdb_lsm_configure(pDb, zConfig); - if( rc!=LSM_OK ){ - testPrintError("lsm_new(): rc=%d\n", rc); - }else{ - rc = lsm_open(pDb, zDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - } - } - - if( rc==LSM_OK ){ - if( bConfig==0 ){ - switch( eOpt ){ - case LSM_INFO_DB_STRUCTURE: - case LSM_INFO_FREELIST: - rc = lsm_info(pDb, eOpt, &z); - break; - case LSM_INFO_ARRAY_STRUCTURE: - case LSM_INFO_ARRAY_PAGES: - case LSM_INFO_PAGE_ASCII_DUMP: - case LSM_INFO_PAGE_HEX_DUMP: - rc = lsm_info(pDb, eOpt, iPg, &z); - break; - default: - assert( !"no chance" ); - } - - if( rc==LSM_OK ){ - printf("%s\n", z ? z : ""); - fflush(stdout); - } - lsm_free(lsm_get_env(pDb), z); - }else{ - int iRes = -1; - lsm_config(pDb, eOpt, &iRes); - printf("%d\n", iRes); - fflush(stdout); - } - } - - lsm_close(pDb); - return rc; - - usage: - testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?"); - return -1; -} diff --git a/ext/lsm1/lsm-test/lsmtest_io.c b/ext/lsm1/lsm-test/lsmtest_io.c deleted file mode 100644 index 7aa5d1094..000000000 --- a/ext/lsm1/lsm-test/lsmtest_io.c +++ /dev/null @@ -1,248 +0,0 @@ - -/* -** SUMMARY -** -** This file implements the 'io' subcommand of the test program. It is used -** for testing the performance of various combinations of write() and fsync() -** system calls. All operations occur on a single file, which may or may not -** exist when a test is started. -** -** A test consists of a series of commands. Each command is either a write -** or an fsync. A write is specified as "<amount>@<offset>", where <amount> -** is the amount of data written, and <offset> is the offset of the file -** to write to. An <amount> or an <offset> is specified as an integer number -** of bytes. Or, if postfixed with a "K", "M" or "G", an integer number of -** KB, MB or GB, respectively. An fsync is simply "S". All commands are -** case-insensitive. -** -** Example test program: -** -** 2M@6M 1492K@4M S 4096@4K S -** -** This program writes 2 MB of data starting at the offset 6MB offset of -** the file, followed by 1492 KB of data written at the 4MB offset of the -** file, followed by a call to fsync(), a write of 4KB of data at byte -** offset 4096, and finally another call to fsync(). -** -** Commands may either be specified on the command line (one command per -** command line argument) or read from stdin. Commands read from stdin -** must be separated by white-space. -** -** COMMAND LINE INVOCATION -** -** The sub-command implemented in this file must be invoked with at least -** two arguments - the path to the file to write to and the page-size to -** use for writing. If there are more than two arguments, then each -** subsequent argument is assumed to be a test command. If there are exactly -** two arguments, the test commands are read from stdin. -** -** A write command does not result in a single call to system call write(). -** Instead, the specified region is written sequentially using one or -** more calls to write(), each of which writes not more than one page of -** data. For example, if the page-size is 4KB, the command "2M@6M" results -** in 512 calls to write(), each of which writes 4KB of data. -** -** EXAMPLES -** -** Two equivalent examples: -** -** $ lsmtest io testfile.db 4KB 2M@6M 1492K@4M S 4096@4K S -** 3544K written in 129 ms -** $ echo "2M@6M 1492K@4M S 4096@4K S" | lsmtest io testfile.db 4096 -** 3544K written in 127 ms -** -*/ - -#include "lsmtest.h" - -typedef struct IoContext IoContext; - -struct IoContext { - int fd; - int nWrite; -}; - -/* -** As isspace(3) -*/ -static int safe_isspace(char c){ - if( c&0x80) return 0; - return isspace(c); -} - -/* -** As isdigit(3) -*/ -static int safe_isdigit(char c){ - if( c&0x80) return 0; - return isdigit(c); -} - -static i64 getNextSize(char *zIn, char **pzOut, int *pRc){ - i64 iRet = 0; - if( *pRc==0 ){ - char *z = zIn; - - if( !safe_isdigit(*z) ){ - *pRc = 1; - return 0; - } - - /* Process digits */ - while( safe_isdigit(*z) ){ - iRet = iRet*10 + (*z - '0'); - z++; - } - - /* Process suffix */ - switch( *z ){ - case 'k': case 'K': - iRet = iRet * 1024; - z++; - break; - - case 'm': case 'M': - iRet = iRet * 1024 * 1024; - z++; - break; - - case 'g': case 'G': - iRet = iRet * 1024 * 1024 * 1024; - z++; - break; - } - - if( pzOut ) *pzOut = z; - } - return iRet; -} - -static int doOneCmd( - IoContext *pCtx, - u8 *aData, - int pgsz, - char *zCmd, - char **pzOut -){ - char c; - char *z = zCmd; - - while( safe_isspace(*z) ) z++; - c = *z; - - if( c==0 ){ - if( pzOut ) *pzOut = z; - return 0; - } - - if( c=='s' || c=='S' ){ - if( pzOut ) *pzOut = &z[1]; - return fdatasync(pCtx->fd); - } - - if( safe_isdigit(c) ){ - i64 iOff = 0; - int nByte = 0; - int rc = 0; - int nPg; - int iPg; - - nByte = (int)getNextSize(z, &z, &rc); - if( rc || *z!='@' ) goto bad_command; - z++; - iOff = getNextSize(z, &z, &rc); - if( rc || (safe_isspace(*z)==0 && *z!='\0') ) goto bad_command; - if( pzOut ) *pzOut = z; - - nPg = (nByte+pgsz-1) / pgsz; - lseek(pCtx->fd, (off_t)iOff, SEEK_SET); - for(iPg=0; iPg<nPg; iPg++){ - write(pCtx->fd, aData, pgsz); - } - pCtx->nWrite += nByte/1024; - - return 0; - } - - bad_command: - testPrintError("unrecognized command: %s", zCmd); - return 1; -} - -static int readStdin(char **pzOut){ - int nAlloc = 128; - char *zOut = 0; - int nOut = 0; - - while( !feof(stdin) ){ - int nRead; - - nAlloc = nAlloc*2; - zOut = realloc(zOut, nAlloc); - nRead = fread(&zOut[nOut], 1, nAlloc-nOut-1, stdin); - - if( nRead==0 ) break; - nOut += nRead; - zOut[nOut] = '\0'; - } - - *pzOut = zOut; - return 0; -} - -int do_io(int nArg, char **azArg){ - IoContext ctx; - int pgsz; - char *zFile; - char *zPgsz; - int i; - int rc = 0; - - char *zStdin = 0; - char *z; - - u8 *aData; - - memset(&ctx, 0, sizeof(IoContext)); - if( nArg<2 ){ - testPrintUsage("FILE PGSZ ?CMD-1 ...?"); - return -1; - } - zFile = azArg[0]; - zPgsz = azArg[1]; - - pgsz = (int)getNextSize(zPgsz, 0, &rc); - if( pgsz<=0 ){ - testPrintError("Ridiculous page size: %d", pgsz); - return -1; - } - aData = malloc(pgsz); - memset(aData, 0x77, pgsz); - - ctx.fd = open(zFile, O_RDWR|O_CREAT|_O_BINARY, 0644); - if( ctx.fd<0 ){ - perror("open: "); - return -1; - } - - if( nArg==2 ){ - readStdin(&zStdin); - testTimeInit(); - z = zStdin; - while( *z && rc==0 ){ - rc = doOneCmd(&ctx, aData, pgsz, z, &z); - } - }else{ - testTimeInit(); - for(i=2; i<nArg; i++){ - rc = doOneCmd(&ctx, aData, pgsz, azArg[i], 0); - } - } - - printf("%dK written in %d ms\n", ctx.nWrite, testTimeGet()); - - free(zStdin); - close(ctx.fd); - - return 0; -} diff --git a/ext/lsm1/lsm-test/lsmtest_main.c b/ext/lsm1/lsm-test/lsmtest_main.c deleted file mode 100644 index f4a3ac0d5..000000000 --- a/ext/lsm1/lsm-test/lsmtest_main.c +++ /dev/null @@ -1,1548 +0,0 @@ - -#include "lsmtest.h" -#include <sqlite3.h> - -void test_failed(){ - assert( 0 ); - return; -} - -#define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__) -static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){ - if( rc ){ - *pRc = rc; - fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc); - test_failed(); - } -} - -static int lsm_memcmp(u8 *a, u8 *b, int c){ - int i; - for(i=0; i<c; i++){ - if( a[i]!=b[i] ) return a[i] - b[i]; - } - return 0; -} - -/* -** A test utility function. -*/ -void testFetch( - TestDb *pDb, /* Database handle */ - void *pKey, int nKey, /* Key to query database for */ - void *pVal, int nVal, /* Expected value */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - void *pDbVal; - int nDbVal; - int rc; - - static int nCall = 0; nCall++; - - rc = tdb_fetch(pDb, pKey, nKey, &pDbVal, &nDbVal); - testSetError(rc); - if( rc==0 && (nVal!=nDbVal || (nVal>0 && lsm_memcmp(pVal, pDbVal, nVal))) ){ - testSetError(1); - } - } -} - -void testWrite( - TestDb *pDb, /* Database handle */ - void *pKey, int nKey, /* Key to query database for */ - void *pVal, int nVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; -static int nCall = 0; -nCall++; - rc = tdb_write(pDb, pKey, nKey, pVal, nVal); - testSetError(rc); - } -} -void testDelete( - TestDb *pDb, /* Database handle */ - void *pKey, int nKey, /* Key to query database for */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; - *pRc = rc = tdb_delete(pDb, pKey, nKey); - testSetError(rc); - } -} -void testDeleteRange( - TestDb *pDb, /* Database handle */ - void *pKey1, int nKey1, - void *pKey2, int nKey2, - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; - *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2); - testSetError(rc); - } -} - -void testBegin(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_begin(pDb, iTrans); - testSetError(rc); - } -} -void testCommit(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_commit(pDb, iTrans); - testSetError(rc); - } -} -#if 0 /* unused */ -static void testRollback(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_rollback(pDb, iTrans); - testSetError(rc); - } -} -#endif - -void testWriteStr( - TestDb *pDb, /* Database handle */ - const char *zKey, /* Key to query database for */ - const char *zVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - int nVal = (zVal ? strlen(zVal) : 0); - testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); -} - -#if 0 /* unused */ -static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){ - testDelete(pDb, (void *)zKey, strlen(zKey), pRc); -} -#endif -void testFetchStr( - TestDb *pDb, /* Database handle */ - const char *zKey, /* Key to query database for */ - const char *zVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - int nVal = (zVal ? strlen(zVal) : 0); - testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); -} - -void testFetchCompare( - TestDb *pControl, - TestDb *pDb, - void *pKey, int nKey, - int *pRc -){ - int rc; - void *pDbVal1; - void *pDbVal2; - int nDbVal1; - int nDbVal2; - - static int nCall = 0; - nCall++; - - rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1); - testSetError(rc); - - rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2); - testSetError(rc); - - if( *pRc==0 - && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1))) - ){ - testSetError(1); - } -} - -typedef struct ScanResult ScanResult; -struct ScanResult { - TestDb *pDb; - - int nRow; - u32 cksum1; - u32 cksum2; - void *pKey1; int nKey1; - void *pKey2; int nKey2; - - int bReverse; - int nPrevKey; - u8 aPrevKey[256]; -}; - -static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){ - int res; - res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2)); - if( res==0 ){ - res = nKey1 - nKey2; - } - return res; -} - -int test_scan_debug = 0; - -static void scanCompareCb( - void *pCtx, - void *pKey, int nKey, - void *pVal, int nVal -){ - ScanResult *p = (ScanResult *)pCtx; - u8 *aKey = (u8 *)pKey; - u8 *aVal = (u8 *)pVal; - int i; - - if( test_scan_debug ){ - printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey); - fflush(stdout); - } -#if 0 - if( test_scan_debug ) printf("%.20s\n", (char *)pVal); -#endif - -#if 0 - /* Check tdb_fetch() matches */ - int rc = 0; - testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc); - assert( rc==0 ); -#endif - - /* Update the checksum data */ - p->nRow++; - for(i=0; i<nKey; i++){ - p->cksum1 += ((int)aKey[i] << (i&0x0F)); - p->cksum2 += p->cksum1; - } - for(i=0; i<nVal; i++){ - p->cksum1 += ((int)aVal[i] << (i&0x0F)); - p->cksum2 += p->cksum1; - } - - /* Check that the delivered row is not out of order. */ - if( nKey<(int)sizeof(p->aPrevKey) ){ - if( p->nPrevKey ){ - int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey); - if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){ - testPrintError("Returned key out of order at %s:%d\n", - __FILE__, __LINE__ - ); - } - } - - p->nPrevKey = nKey; - memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey)); - } - - /* Check that the delivered row is within range. */ - if( p->pKey1 && ( - (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0) - || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey) - )){ - testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__); - } - if( p->pKey2 && ( - (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0) - || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2<nKey) - )){ - testPrintError("Returned key too large at %s:%d\n", __FILE__, __LINE__); - } - -} - -/* -** Scan the contents of the two databases. Check that they match. -*/ -void testScanCompare( - TestDb *pDb1, /* Control (trusted) database */ - TestDb *pDb2, /* Database being tested */ - int bReverse, - void *pKey1, int nKey1, - void *pKey2, int nKey2, - int *pRc -){ - static int nCall = 0; nCall++; - if( *pRc==0 ){ - ScanResult res1; - ScanResult res2; - void *pRes1 = (void *)&res1; - void *pRes2 = (void *)&res2; - - memset(&res1, 0, sizeof(ScanResult)); - memset(&res2, 0, sizeof(ScanResult)); - - res1.pDb = pDb1; - res1.nKey1 = nKey1; res1.pKey1 = pKey1; - res1.nKey2 = nKey2; res1.pKey2 = pKey2; - res1.bReverse = bReverse; - res2.pDb = pDb2; - res2.nKey1 = nKey1; res2.pKey1 = pKey1; - res2.nKey2 = nKey2; res2.pKey2 = pKey2; - res2.bReverse = bReverse; - - tdb_scan(pDb1, pRes1, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb); -if( test_scan_debug ) printf("\n\n\n"); - tdb_scan(pDb2, pRes2, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb); -if( test_scan_debug ) printf("\n\n\n"); - - if( res1.nRow!=res2.nRow - || res1.cksum1!=res2.cksum1 - || res1.cksum2!=res2.cksum2 - ){ - printf("expected: %d %X %X\n", res1.nRow, res1.cksum1, res1.cksum2); - printf("got: %d %X %X\n", res2.nRow, res2.cksum1, res2.cksum2); - testSetError(1); - *pRc = 1; - } - } -} - -void testClose(TestDb **ppDb){ - tdb_close(*ppDb); - *ppDb = 0; -} - -TestDb *testOpen(const char *zSystem, int bClear, int *pRc){ - TestDb *pDb = 0; - if( *pRc==0 ){ - int rc; - rc = tdb_open(zSystem, 0, bClear, &pDb); - if( rc!=0 ){ - testSetError(rc); - *pRc = rc; - } - } - return pDb; -} - -void testReopen(TestDb **ppDb, int *pRc){ - if( *pRc==0 ){ - const char *zLib; - zLib = tdb_library_name(*ppDb); - testClose(ppDb); - *pRc = tdb_open(zLib, 0, 0, ppDb); - } -} - - -#if 0 /* unused */ -static void testSystemSelect(const char *zSys, int *piSel, int *pRc){ - if( *pRc==0 ){ - struct SysName { const char *zName; } *aName; - int nSys; - int i; - - for(nSys=0; tdb_system_name(nSys); nSys++); - aName = malloc(sizeof(struct SysName) * (nSys+1)); - for(i=0; i<=nSys; i++){ - aName[i].zName = tdb_system_name(i); - } - - *pRc = testArgSelect(aName, "db", zSys, piSel); - free(aName); - } -} -#endif - -char *testMallocVPrintf(const char *zFormat, va_list ap){ - int nByte; - va_list copy; - char *zRet; - - __va_copy(copy, ap); - nByte = vsnprintf(0, 0, zFormat, copy); - va_end(copy); - - assert( nByte>=0 ); - zRet = (char *)testMalloc(nByte+1); - vsnprintf(zRet, nByte+1, zFormat, ap); - return zRet; -} - -char *testMallocPrintf(const char *zFormat, ...){ - va_list ap; - char *zRet; - - va_start(ap, zFormat); - zRet = testMallocVPrintf(zFormat, ap); - va_end(ap); - - return zRet; -} - - -/* -** A wrapper around malloc(3). -** -** This function should be used for all allocations made by test procedures. -** It has the following properties: -** -** * Test code may assume that allocations may not fail. -** * Returned memory is always zeroed. -** -** Allocations made using testMalloc() should be freed using testFree(). -*/ -void *testMalloc(int n){ - u8 *p = (u8*)malloc(n + 8); - memset(p, 0, n+8); - *(int*)p = n; - return (void*)&p[8]; -} - -void *testMallocCopy(void *pCopy, int nByte){ - void *pRet = testMalloc(nByte); - memcpy(pRet, pCopy, nByte); - return pRet; -} - -void *testRealloc(void *ptr, int n){ - if( ptr ){ - u8 *p = (u8*)ptr - 8; - int nOrig = *(int*)p; - p = (u8*)realloc(p, n+8); - if( nOrig<n ){ - memset(&p[8+nOrig], 0, n-nOrig); - } - *(int*)p = n; - return (void*)&p[8]; - } - return testMalloc(n); -} - -/* -** Free an allocation made by an earlier call to testMalloc(). -*/ -void testFree(void *ptr){ - if( ptr ){ - u8 *p = (u8*)ptr - 8; - memset(p, 0x55, *(int*)p + 8); - free(p); - } -} - -/* -** String zPattern contains a glob pattern. Return true if zStr matches -** the pattern, or false if it does not. -*/ -int testGlobMatch(const char *zPattern, const char *zStr){ - int i = 0; - int j = 0; - - while( zPattern[i] ){ - char p = zPattern[i]; - - if( p=='*' || p=='%' ){ - do { - if( testGlobMatch(&zPattern[i+1], &zStr[j]) ) return 1; - }while( zStr[j++] ); - return 0; - } - - if( zStr[j]==0 || (p!='?' && p!=zStr[j]) ){ - /* Match failed. */ - return 0; - } - - j++; - i++; - } - - return (zPattern[i]==0 && zStr[j]==0); -} - -/* -** End of test utilities -**************************************************************************/ - -int do_test(int nArg, char **azArg){ - int j; - int rc; - int nFail = 0; - const char *zPattern = 0; - - if( nArg>1 ){ - testPrintError("Usage: test ?PATTERN?\n"); - return 1; - } - if( nArg==1 ){ - zPattern = azArg[0]; - } - - for(j=0; tdb_system_name(j); j++){ - rc = 0; - - test_data_1(tdb_system_name(j), zPattern, &rc); - test_data_2(tdb_system_name(j), zPattern, &rc); - test_data_3(tdb_system_name(j), zPattern, &rc); - test_data_4(tdb_system_name(j), zPattern, &rc); - test_rollback(tdb_system_name(j), zPattern, &rc); - test_mc(tdb_system_name(j), zPattern, &rc); - test_mt(tdb_system_name(j), zPattern, &rc); - - if( rc ) nFail++; - } - - rc = 0; - test_oom(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - test_api(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - do_crash_test(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - do_writer_crash_test(zPattern, &rc); - if( rc ) nFail++; - - return (nFail!=0); -} - -static lsm_db *configure_lsm_db(TestDb *pDb){ - lsm_db *pLsm; - pLsm = tdb_lsm(pDb); - if( pLsm ){ - tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4"); - } - return pLsm; -} - -typedef struct WriteHookEvent WriteHookEvent; -struct WriteHookEvent { - i64 iOff; - int nData; - int nUs; -}; -WriteHookEvent prev = {0, 0, 0}; - -static void flushPrev(FILE *pOut){ - if( prev.nData ){ - fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs); - prev.nData = 0; - } -} - -#if 0 /* unused */ -static void do_speed_write_hook2( - void *pCtx, - int bLog, - i64 iOff, - int nData, - int nUs -){ - FILE *pOut = (FILE *)pCtx; - if( bLog ) return; - - if( prev.nData && nData && iOff==prev.iOff+prev.nData ){ - prev.nData += nData; - prev.nUs += nUs; - }else{ - flushPrev(pOut); - if( nData==0 ){ - fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs); - }else{ - prev.iOff = iOff; - prev.nData = nData; - prev.nUs = nUs; - } - } -} -#endif - -#define ST_REPEAT 0 -#define ST_WRITE 1 -#define ST_PAUSE 2 -#define ST_FETCH 3 -#define ST_SCAN 4 -#define ST_NSCAN 5 -#define ST_KEYSIZE 6 -#define ST_VALSIZE 7 -#define ST_TRANS 8 - - -static void print_speed_test_help(){ - printf( -"\n" -"Repeat the following $repeat times:\n" -" 1. Insert $write key-value pairs. One transaction for each write op.\n" -" 2. Pause for $pause ms.\n" -" 3. Perform $fetch queries on the database.\n" -"\n" -" Keys are $keysize bytes in size. Values are $valsize bytes in size\n" -" Both keys and values are pseudo-randomly generated\n" -"\n" -"Options are:\n" -" -repeat $repeat (default value 10)\n" -" -write $write (default value 10000)\n" -" -pause $pause (default value 0)\n" -" -fetch $fetch (default value 0)\n" -" -keysize $keysize (default value 12)\n" -" -valsize $valsize (default value 100)\n" -" -system $system (default value \"lsm\")\n" -" -trans $trans (default value 0)\n" -"\n" -); -} - -int do_speed_test2(int nArg, char **azArg){ - struct Option { - const char *zOpt; - int eVal; - int iDefault; - } aOpt[] = { - { "-repeat", ST_REPEAT, 10}, - { "-write", ST_WRITE, 10000}, - { "-pause", ST_PAUSE, 0}, - { "-fetch", ST_FETCH, 0}, - { "-scan", ST_SCAN, 0}, - { "-nscan", ST_NSCAN, 0}, - { "-keysize", ST_KEYSIZE, 12}, - { "-valsize", ST_VALSIZE, 100}, - { "-trans", ST_TRANS, 0}, - { "-system", -1, 0}, - { "help", -2, 0}, - {0, 0, 0} - }; - int i; - int aParam[9]; - int rc = 0; - int bReadonly = 0; - int nContent = 0; - - TestDb *pDb; - Datasource *pData; - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 }; - char *zSystem = ""; - int bLsm = 1; - FILE *pLog = 0; - -#ifdef NDEBUG - /* If NDEBUG is defined, disable the dynamic memory related checks in - ** lsmtest_mem.c. They slow things down. */ - testMallocUninstall(tdb_lsm_env()); -#endif - - /* Initialize aParam[] with default values. */ - for(i=0; i<ArraySize(aOpt); i++){ - if( aOpt[i].zOpt ) aParam[aOpt[i].eVal] = aOpt[i].iDefault; - } - - /* Process the command line switches. */ - for(i=0; i<nArg; i+=2){ - int iSel; - rc = testArgSelect(aOpt, "switch", azArg[i], &iSel); - if( rc ){ - return rc; - } - if( aOpt[iSel].eVal==-2 ){ - print_speed_test_help(); - return 0; - } - if( i+1==nArg ){ - testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt); - return 1; - } - if( aOpt[iSel].eVal>=0 ){ - aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]); - }else{ - zSystem = azArg[i+1]; - bLsm = 0; -#if 0 - for(j=0; zSystem[j]; j++){ - if( zSystem[j]=='=' ) bLsm = 1; - } -#endif - } - } - - printf("#"); - for(i=0; i<ArraySize(aOpt); i++){ - if( aOpt[i].zOpt ){ - if( aOpt[i].eVal>=0 ){ - printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]); - }else if( aOpt[i].eVal==-1 ){ - printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem); - } - } - } - printf("\n"); - - defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE]; - defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE]; - pData = testDatasourceNew(&defn); - - if( aParam[ST_WRITE]==0 ){ - bReadonly = 1; - } - - if( bLsm ){ - rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb); - }else{ - pDb = testOpen(zSystem, !bReadonly, &rc); - } - if( rc!=0 ) return rc; - if( bReadonly ){ - nContent = testCountDatabase(pDb); - } - -#if 0 - pLog = fopen("/tmp/speed.log", "w"); - tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog); -#endif - - for(i=0; i<aParam[ST_REPEAT] && rc==0; i++){ - int msWrite, msFetch; - int iFetch; - int nWrite = aParam[ST_WRITE]; - - if( bReadonly ){ - msWrite = 0; - }else{ - testTimeInit(); - - if( aParam[ST_TRANS] ) testBegin(pDb, 2, &rc); - testWriteDatasourceRange(pDb, pData, i*nWrite, nWrite, &rc); - if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc); - - msWrite = testTimeGet(); - nContent += nWrite; - } - - if( aParam[ST_PAUSE] ){ - if( aParam[ST_PAUSE]/1000 ) sleep(aParam[ST_PAUSE]/1000); - if( aParam[ST_PAUSE]%1000 ) usleep(1000 * (aParam[ST_PAUSE]%1000)); - } - - if( aParam[ST_FETCH] ){ - testTimeInit(); - if( aParam[ST_TRANS] ) testBegin(pDb, 1, &rc); - for(iFetch=0; iFetch<aParam[ST_FETCH]; iFetch++){ - int iKey = testPrngValue(i*nWrite+iFetch) % nContent; -#ifndef NDEBUG - testDatasourceFetch(pDb, pData, iKey, &rc); -#else - void *pKey; int nKey; /* Database key to query for */ - void *pVal; int nVal; /* Result of query */ - - testDatasourceEntry(pData, iKey, &pKey, &nKey, 0, 0); - rc = tdb_fetch(pDb, pKey, nKey, &pVal, &nVal); - if( rc==0 && nVal<0 ) rc = 1; - if( rc ) break; -#endif - } - if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc); - msFetch = testTimeGet(); - }else{ - msFetch = 0; - } - - if( i==(aParam[ST_REPEAT]-1) ){ - testTimeInit(); - testClose(&pDb); - msWrite += testTimeGet(); - } - - printf("%d %d %d\n", i, msWrite, msFetch); - fflush(stdout); - } - - testClose(&pDb); - testDatasourceFree(pData); - - if( pLog ){ - flushPrev(pLog); - fclose(pLog); - } - return rc; -} - -int do_speed_tests(int nArg, char **azArg){ - - struct DbSystem { - const char *zLibrary; - const char *zColor; - } aSys[] = { - { "sqlite3", "black" }, - { "leveldb", "blue" }, - { "lsm", "red" }, - { "lsm_mt2", "orange" }, - { "lsm_mt3", "purple" }, - { "kyotocabinet", "green" }, - {0, 0} - }; - - int i; - int j; - int rc; - int nSleep = 0; /* ms of rest allowed between INSERT tests */ - int nRow = 0; /* Number of rows to insert into database */ - int nStep; /* Measure INSERT time after this many rows */ - int nSelStep; /* Measure SELECT time after this many rows */ - int nSelTest; /* Number of SELECTs to run for timing */ - int doReadTest = 1; - int doWriteTest = 1; - - int *aTime; /* INSERT timing data */ - int *aWrite; /* Writes per nStep inserts */ - int *aSelTime; /* SELECT timing data */ - int isFirst = 1; - int bSleep = 0; - - /* File to write gnuplot script to. */ - const char *zOut = "lsmtest_speed.gnuplot"; - - u32 sys_mask = 0; - - testMallocUninstall(tdb_lsm_env()); - - for(i=0; i<nArg; i++){ - struct Opt { - const char *zOpt; - int isSwitch; - } aOpt[] = { - { "sqlite3" , 0}, - { "leveldb" , 0}, - { "lsm" , 0}, - { "lsm_mt2" , 0}, - { "lsm_mt3" , 0}, - { "kyotocabinet" , 0}, - { "-rows" , 1}, - { "-sleep" , 2}, - { "-testmode" , 3}, - { "-out" , 4}, - { 0, 0} - }; - int iSel; - - rc = testArgSelect(aOpt, "argument", azArg[i], &iSel); - if( rc ) return rc; - - if( aOpt[iSel].isSwitch ){ - i++; - - if( i>=nArg ){ - testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt); - return 1; - } - if( aOpt[iSel].isSwitch==1 ){ - nRow = atoi(azArg[i]); - } - if( aOpt[iSel].isSwitch==2 ){ - nSleep = atoi(azArg[i]); - } - if( aOpt[iSel].isSwitch==3 ){ - struct Mode { - const char *zMode; - int doReadTest; - int doWriteTest; - } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}}; - int iMode; - rc = testArgSelect(aMode, "option", azArg[i], &iMode); - if( rc ) return rc; - doReadTest = aMode[iMode].doReadTest; - doWriteTest = aMode[iMode].doWriteTest; - } - if( aOpt[iSel].isSwitch==4 ){ - /* The "-out FILE" switch. This option is used to specify a file to - ** write the gnuplot script to. */ - zOut = azArg[i]; - } - }else{ - /* A db name */ - rc = testArgSelect(aOpt, "system", azArg[i], &iSel); - if( rc ) return rc; - sys_mask |= (1<<iSel); - } - } - - if( sys_mask==0 ) sys_mask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3); - nRow = MAX(nRow, 100000); - nStep = nRow/100; - nSelStep = nRow/10; - nSelTest = (nSelStep > 100000) ? 100000 : nSelStep; - - aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep); - aWrite = malloc(sizeof(int) * nRow/nStep); - aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep); - - /* This loop collects the INSERT speed data. */ - if( doWriteTest ){ - printf("Writing output to file \"%s\".\n", zOut); - - for(j=0; aSys[j].zLibrary; j++){ - FILE *pLog = 0; - TestDb *pDb; /* Database being tested */ - lsm_db *pLsm; - int iDot = 0; - - if( ((1<<j)&sys_mask)==0 ) continue; - if( bSleep && nSleep ) sqlite3_sleep(nSleep); - bSleep = 1; - - testCaseBegin(&rc, 0, "speed.insert.%s", aSys[j].zLibrary); - - rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb); - if( rc ) return rc; - - pLsm = configure_lsm_db(pDb); -#if 0 - pLog = fopen("/tmp/speed.log", "w"); - tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog); -#endif - - testTimeInit(); - for(i=0; i<nRow; i+=nStep){ - int iStep; - int nWrite1 = 0, nWrite2 = 0; - testCaseProgress(i, nRow, testCaseNDot(), &iDot); - if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite1); - for(iStep=0; iStep<nStep; iStep++){ - u32 aKey[4]; /* 16-byte key */ - u32 aVal[25]; /* 100 byte value */ - testPrngArray(i+iStep, aKey, ArraySize(aKey)); - testPrngArray(i+iStep, aVal, ArraySize(aVal)); - rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal)); - } - aTime[(j*nRow+i)/nStep] = testTimeGet(); - if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite2); - aWrite[i/nStep] = nWrite2 - nWrite1; - } - - tdb_close(pDb); - if( pLog ) fclose(pLog); - testCaseFinish(rc); - } - } - - /* This loop collects the SELECT speed data. */ - if( doReadTest ){ - for(j=0; aSys[j].zLibrary; j++){ - int iDot = 0; - TestDb *pDb; /* Database being tested */ - - if( ((1<<j)&sys_mask)==0 ) continue; - if( bSleep && nSleep ) sqlite3_sleep(nSleep); - bSleep = 1; - - testCaseBegin(&rc, 0, "speed.select.%s", aSys[j].zLibrary); - - if( doWriteTest ){ - rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb); - if( rc ) return rc; - configure_lsm_db(pDb); - - for(i=0; i<nRow; i+=nSelStep){ - int iStep; - int iSel; - testCaseProgress(i, nRow, testCaseNDot(), &iDot); - for(iStep=0; iStep<nSelStep; iStep++){ - u32 aKey[4]; /* 16-byte key */ - u32 aVal[25]; /* 100 byte value */ - testPrngArray(i+iStep, aKey, ArraySize(aKey)); - testPrngArray(i+iStep, aVal, ArraySize(aVal)); - rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal)); - } - - testTimeInit(); - for(iSel=0; iSel<nSelTest; iSel++){ - void *pDummy; - int nDummy; - u32 iKey; - u32 aKey[4]; /* 16-byte key */ - - iKey = testPrngValue(iSel) % (i+nSelStep); - testPrngArray(iKey, aKey, ArraySize(aKey)); - rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy); - } - aSelTime[(j*nRow+i)/nSelStep] = testTimeGet(); - tdb_fetch(pDb, 0, 0, 0, 0); - } - }else{ - int t; - int iSel; - - rc = tdb_open(aSys[j].zLibrary, 0, 0, &pDb); - configure_lsm_db(pDb); - - testTimeInit(); - for(iSel=0; rc==LSM_OK && iSel<nSelTest; iSel++){ - void *pDummy; - int nDummy; - u32 iKey; - u32 aKey[4]; /* 16-byte key */ -#ifndef NDEBUG - u32 aVal[25]; /* 100 byte value */ -#endif - - testCaseProgress(iSel, nSelTest, testCaseNDot(), &iDot); - - iKey = testPrngValue(iSel) % nRow; - testPrngArray(iKey, aKey, ArraySize(aKey)); - rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy); - -#ifndef NDEBUG - testPrngArray(iKey, aVal, ArraySize(aVal)); - assert( nDummy==100 && memcmp(aVal, pDummy, 100)==0 ); -#endif - } - if( rc!=LSM_OK ) return rc; - - t = testTimeGet(); - tdb_fetch(pDb, 0, 0, 0, 0); - - printf("%s: %d selects/second\n", - aSys[j].zLibrary, (int)((double)nSelTest*1000.0/t) - ); - } - - tdb_close(pDb); - testCaseFinish(rc); - } - } - - - if( doWriteTest ){ - FILE *pOut = fopen(zOut, "w"); - if( !pOut ){ - printf("fopen(\"%s\", \"w\"): %s\n", zOut, strerror(errno)); - return 1; - } - - fprintf(pOut, "set xlabel \"Rows Inserted\"\n"); - fprintf(pOut, "set ylabel \"Inserts per second\"\n"); - if( doReadTest ){ - fprintf(pOut, "set y2label \"Selects per second\"\n"); - }else if( sys_mask==(1<<2) ){ - fprintf(pOut, "set y2label \"Page writes per insert\"\n"); - } - fprintf(pOut, "set yrange [0:*]\n"); - fprintf(pOut, "set y2range [0:*]\n"); - fprintf(pOut, "set xrange [%d:*]\n", MAX(nStep, nRow/20) ); - fprintf(pOut, "set ytics nomirror\n"); - fprintf(pOut, "set y2tics nomirror\n"); - fprintf(pOut, "set key box lw 0.01\n"); - fprintf(pOut, "plot "); - - for(j=0; aSys[j].zLibrary; j++){ - if( (1<<j)&sys_mask ){ - const char *zLib = aSys[j].zLibrary; - fprintf(pOut, "%s\"-\" ti \"%s INSERT\" with lines lc rgb \"%s\" ", - (isFirst?"":", "), zLib, aSys[j].zColor - ); - if( doReadTest ){ - fprintf(pOut, ", \"-\" ti \"%s SELECT\" " - "axis x1y2 with points lw 3 lc rgb \"%s\"" - , zLib, aSys[j].zColor - ); - } - isFirst = 0; - } - } - - assert( strcmp(aSys[2].zLibrary, "lsm")==0 ); - if( sys_mask==(1<<2) && !doReadTest ){ - fprintf(pOut, ", \"-\" ti \"lsm pages written\" " - "axis x1y2 with boxes lw 1 lc rgb \"grey\"" - ); - } - - fprintf(pOut, "\n"); - - for(j=0; aSys[j].zLibrary; j++){ - if( ((1<<j)&sys_mask)==0 ) continue; - fprintf(pOut, "# Rows Inserts per second\n"); - for(i=0; i<nRow; i+=nStep){ - int iTime = aTime[(j*nRow+i)/nStep]; - int ips = (int)((i+nStep)*1000.0 / (double)iTime); - fprintf(pOut, "%d %d\n", i+nStep, ips); - } - fprintf(pOut, "end\n"); - - if( doReadTest ){ - fprintf(pOut, "# Rows Selects per second\n"); - for(i=0; i<nRow; i+=nSelStep){ - int sps = (int)(nSelTest*1000.0/(double)aSelTime[(j*nRow+i)/nSelStep]); - fprintf(pOut, "%d %d\n", i+nSelStep, sps); - } - fprintf(pOut, "end\n"); - }else if( sys_mask==(1<<2) ){ - for(i=0; i<(nRow/nStep); i++){ - fprintf(pOut, "%d %f\n", i*nStep, (double)aWrite[i] / (double)nStep); - } - fprintf(pOut, "end\n"); - } - } - - fprintf(pOut, "pause -1\n"); - fclose(pOut); - } - - free(aTime); - free(aSelTime); - free(aWrite); - testMallocInstall(tdb_lsm_env()); - return 0; -} - -/* -** Usage: lsmtest random ?N? -** -** This command prints a sequence of zero or more numbers from the PRNG -** system to stdout. If the "N" argument is missing, values the first 10 -** values (i=0, i=1, ... i=9) are printed. Otherwise, the first N. -** -** This was added to verify that the PRNG values do not change between -** runs of the lsmtest program. -*/ -int do_random_tests(int nArg, char **azArg){ - int i; - int nRand; - if( nArg==0 ){ - nRand = 10; - }else if( nArg==1 ){ - nRand = atoi(azArg[0]); - }else{ - testPrintError("Usage: random ?N?\n"); - return -1; - } - for(i=0; i<nRand; i++){ - printf("0x%x\n", testPrngValue(i)); - } - return 0; -} - -static int testFormatSize(char *aBuf, int nBuf, i64 nByte){ - int res; - if( nByte<(1<<10) ){ - res = snprintf(aBuf, nBuf, "%d byte", (int)nByte); - }else if( nByte<(1<<20) ){ - res = snprintf(aBuf, nBuf, "%dK", (int)(nByte/(1<<10))); - }else{ - res = snprintf(aBuf, nBuf, "%dM", (int)(nByte/(1<<20))); - } - return res; -} - -static i64 testReadSize(char *z){ - int n = strlen(z); - char c = z[n-1]; - i64 nMul = 1; - - switch( c ){ - case 'g': case 'G': - nMul = (1<<30); - break; - - case 'm': case 'M': - nMul = (1<<20); - break; - - case 'k': case 'K': - nMul = (1<<10); - break; - - default: - nMul = 1; - } - - return nMul * (i64)atoi(z); -} - -/* -** Usage: lsmtest writespeed FILESIZE BLOCKSIZE SYNCSIZE -*/ -static int do_writer_test(int nArg, char **azArg){ - int nBlock; - int nSize; - int i; - int fd; - int ms; - char aFilesize[32]; - char aBlockSize[32]; - - char *aPage; - int *aOrder; - int nSync; - - i64 filesize; - i64 blocksize; - i64 syncsize; - int nPage = 4096; - - /* How long to sleep before running a trial (in ms). */ -#if 0 - const int nSleep = 10000; -#endif - const int nSleep = 0; - - if( nArg!=3 ){ - testPrintUsage("FILESIZE BLOCKSIZE SYNCSIZE"); - return -1; - } - - filesize = testReadSize(azArg[0]); - blocksize = testReadSize(azArg[1]); - syncsize = testReadSize(azArg[2]); - - nBlock = (int)(filesize / blocksize); - nSize = (int)blocksize; - nSync = (int)(syncsize / blocksize); - - aPage = (char *)malloc(4096); - aOrder = (int *)malloc(nBlock * sizeof(int)); - for(i=0; i<nBlock; i++) aOrder[i] = i; - for(i=0; i<(nBlock*25); i++){ - int tmp; - u32 a = testPrngValue(i); - u32 b = testPrngValue(a); - a = a % nBlock; - b = b % nBlock; - tmp = aOrder[a]; - aOrder[a] = aOrder[b]; - aOrder[b] = tmp; - } - - testFormatSize(aFilesize, sizeof(aFilesize), (i64)nBlock * (i64)nSize); - testFormatSize(aBlockSize, sizeof(aFilesize), nSize); - - printf("Testing writing a %s file using %s blocks. ", aFilesize, aBlockSize); - if( nSync==1 ){ - printf("Sync after each block.\n"); - }else{ - printf("Sync after each %d blocks.\n", nSync); - } - - printf("Preparing file... "); - fflush(stdout); - unlink("writer.out"); - fd = open("writer.out", O_RDWR|O_CREAT|_O_BINARY, 0664); - if( fd<0 ){ - testPrintError("open(): %d - %s\n", errno, strerror(errno)); - return -1; - } - testTimeInit(); - for(i=0; i<nBlock; i++){ - int iPg; - memset(aPage, i&0xFF, nPage); - for(iPg=0; iPg<(nSize/nPage); iPg++){ - write(fd, aPage, nPage); - } - } - fsync(fd); - printf("ok (%d ms)\n", testTimeGet()); - - for(i=0; i<5; i++){ - int j; - - sqlite3_sleep(nSleep); - printf("Now writing sequentially... "); - fflush(stdout); - - lseek(fd, 0, SEEK_SET); - testTimeInit(); - for(j=0; j<nBlock; j++){ - int iPg; - if( ((j+1)%nSync)==0 ) fdatasync(fd); - memset(aPage, j&0xFF, nPage); - for(iPg=0; iPg<(nSize/nPage); iPg++){ - write(fd, aPage, nPage); - } - } - fdatasync(fd); - ms = testTimeGet(); - printf("%d ms\n", ms); - sqlite3_sleep(nSleep); - printf("Now in an arbitrary order... "); - - fflush(stdout); - testTimeInit(); - for(j=0; j<nBlock; j++){ - int iPg; - if( ((j+1)%nSync)==0 ) fdatasync(fd); - lseek(fd, aOrder[j]*nSize, SEEK_SET); - memset(aPage, j&0xFF, nPage); - for(iPg=0; iPg<(nSize/nPage); iPg++){ - write(fd, aPage, nPage); - } - } - fdatasync(fd); - ms = testTimeGet(); - printf("%d ms\n", ms); - } - - close(fd); - free(aPage); - free(aOrder); - - return 0; -} - -static void do_insert_work_hook(lsm_db *db, void *p){ - char *z = 0; - lsm_info(db, LSM_INFO_DB_STRUCTURE, &z); - if( z ){ - printf("%s\n", z); - fflush(stdout); - lsm_free(lsm_get_env(db), z); - } - - unused_parameter(p); -} - -typedef struct InsertWriteHook InsertWriteHook; -struct InsertWriteHook { - FILE *pOut; - int bLog; - i64 iOff; - int nData; -}; - -static void flushHook(InsertWriteHook *pHook){ - if( pHook->nData ){ - fprintf(pHook->pOut, "write %s %d %d\n", - (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData - ); - pHook->nData = 0; - fflush(pHook->pOut); - } -} - -static void do_insert_write_hook( - void *pCtx, - int bLog, - i64 iOff, - int nData, - int nUs -){ - InsertWriteHook *pHook = (InsertWriteHook *)pCtx; - if( bLog ) return; - - if( nData==0 ){ - flushHook(pHook); - fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db")); - }else if( pHook->nData - && bLog==pHook->bLog - && iOff==(pHook->iOff+pHook->nData) - ){ - pHook->nData += nData; - }else{ - flushHook(pHook); - pHook->bLog = bLog; - pHook->iOff = iOff; - pHook->nData = nData; - } -} - -static int do_replay(int nArg, char **azArg){ - char aBuf[4096]; - FILE *pInput; - FILE *pClose = 0; - const char *zDb; - - lsm_env *pEnv; - lsm_file *pOut; - int rc; - - if( nArg!=2 ){ - testPrintError("Usage: replay WRITELOG FILE\n"); - return 1; - } - - if( strcmp(azArg[0], "-")==0 ){ - pInput = stdin; - }else{ - pClose = pInput = fopen(azArg[0], "r"); - } - zDb = azArg[1]; - pEnv = tdb_lsm_env(); - rc = pEnv->xOpen(pEnv, zDb, 0, &pOut); - if( rc!=LSM_OK ) return rc; - - while( feof(pInput)==0 ){ - char zLine[80]; - fgets(zLine, sizeof(zLine)-1, pInput); - zLine[sizeof(zLine)-1] = '\0'; - - if( 0==memcmp("sync db", zLine, 7) ){ - rc = pEnv->xSync(pOut); - if( rc!=0 ) break; - }else{ - int iOff; - int nData; - int nMatch; - nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData); - if( nMatch==2 ){ - int i; - for(i=0; i<nData; i+=sizeof(aBuf)){ - memset(aBuf, i&0xFF, sizeof(aBuf)); - rc = pEnv->xWrite(pOut, iOff+i, aBuf, sizeof(aBuf)); - if( rc!=0 ) break; - } - } - } - } - if( pClose ) fclose(pClose); - pEnv->xClose(pOut); - - return rc; -} - -static int do_insert(int nArg, char **azArg){ - const char *zDb = "lsm"; - TestDb *pDb = 0; - int i; - int rc; - const int nRow = 1 * 1000 * 1000; - - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 }; - Datasource *pData = 0; - - if( nArg>1 ){ - testPrintError("Usage: insert ?DATABASE?\n"); - return 1; - } - if( nArg==1 ){ zDb = azArg[0]; } - - testMallocUninstall(tdb_lsm_env()); - for(i=0; zDb[i] && zDb[i]!='='; i++); - if( zDb[i] ){ - rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb); - }else{ - rc = tdb_open(zDb, 0, 1, &pDb); - } - - if( rc!=0 ){ - testPrintError("Error opening db \"%s\": %d\n", zDb, rc); - }else{ - InsertWriteHook hook; - memset(&hook, 0, sizeof(hook)); - hook.pOut = fopen("writelog.txt", "w"); - - pData = testDatasourceNew(&defn); - tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0); - tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook); - - if( rc==0 ){ - for(i=0; i<nRow; i++){ - void *pKey; int nKey; /* Database key to insert */ - void *pVal; int nVal; /* Database value to insert */ - testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); - tdb_write(pDb, pKey, nKey, pVal, nVal); - } - } - - testDatasourceFree(pData); - tdb_close(pDb); - flushHook(&hook); - fclose(hook.pOut); - } - testMallocInstall(tdb_lsm_env()); - - return rc; -} - -static int st_do_show(int a, char **b) { return do_show(a, b); } -static int st_do_work(int a, char **b) { return do_work(a, b); } -static int st_do_io(int a, char **b) { return do_io(a, b); } - -#ifdef __linux__ -#include <sys/time.h> -#include <sys/resource.h> - -static void lsmtest_rusage_report(void){ - struct rusage r; - memset(&r, 0, sizeof(r)); - - getrusage(RUSAGE_SELF, &r); - printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n", - (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock - ); -} -#else -static void lsmtest_rusage_report(void){ - /* no-op */ -} -#endif - -int main(int argc, char **argv){ - struct TestFunc { - const char *zName; - int bRusageReport; - int (*xFunc)(int, char **); - } aTest[] = { - {"random", 1, do_random_tests}, - {"writespeed", 1, do_writer_test}, - {"io", 1, st_do_io}, - - {"insert", 1, do_insert}, - {"replay", 1, do_replay}, - - {"speed", 1, do_speed_tests}, - {"speed2", 1, do_speed_test2}, - {"show", 0, st_do_show}, - {"work", 1, st_do_work}, - {"test", 1, do_test}, - - {0, 0} - }; - int rc; /* Return Code */ - int iFunc; /* Index into aTest[] */ - - int nLeakAlloc = 0; /* Allocations leaked by lsm */ - int nLeakByte = 0; /* Bytes leaked by lsm */ - -#ifdef LSM_DEBUG_MEM - FILE *pReport = 0; /* lsm malloc() report file */ - const char *zReport = "malloc.txt generated"; -#else - const char *zReport = "malloc.txt NOT generated"; -#endif - - testMallocInstall(tdb_lsm_env()); - - if( argc<2 ){ - testPrintError("Usage: %s sub-command ?args...?\n", argv[0]); - return -1; - } - - /* Initialize error reporting */ - testErrorInit(argc, argv); - - /* Initialize PRNG system */ - testPrngInit(); - - rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc); - if( rc==0 ){ - rc = aTest[iFunc].xFunc(argc-2, &argv[2]); - } - -#ifdef LSM_DEBUG_MEM - pReport = fopen("malloc.txt", "w"); - testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport); - fclose(pReport); -#else - testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0); -#endif - - if( nLeakAlloc ){ - testPrintError("Leaked %d bytes in %d allocations (%s)\n", - nLeakByte, nLeakAlloc, zReport - ); - if( rc==0 ) rc = -1; - } - testMallocUninstall(tdb_lsm_env()); - - if( aTest[iFunc].bRusageReport ){ - lsmtest_rusage_report(); - } - return rc; -} diff --git a/ext/lsm1/lsm-test/lsmtest_mem.c b/ext/lsm1/lsm-test/lsmtest_mem.c deleted file mode 100644 index 4c35e849f..000000000 --- a/ext/lsm1/lsm-test/lsmtest_mem.c +++ /dev/null @@ -1,409 +0,0 @@ - -#include <stdio.h> -#include <assert.h> -#include <string.h> - -#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) - -#define MIN(x,y) ((x)<(y) ? (x) : (y)) - -typedef unsigned int u32; -typedef unsigned char u8; -typedef long long int i64; -typedef unsigned long long int u64; - -#if defined(__GLIBC__) && defined(LSM_DEBUG_MEM) - extern int backtrace(void**,int); - extern void backtrace_symbols_fd(void*const*,int,int); -# define TM_BACKTRACE 12 -#else -# define backtrace(A,B) 1 -# define backtrace_symbols_fd(A,B,C) -#endif - - -typedef struct TmBlockHdr TmBlockHdr; -typedef struct TmAgg TmAgg; -typedef struct TmGlobal TmGlobal; - -struct TmGlobal { - /* Linked list of all currently outstanding allocations. And a table of - ** all allocations, past and present, indexed by backtrace() info. */ - TmBlockHdr *pFirst; -#ifdef TM_BACKTRACE - TmAgg *aHash[10000]; -#endif - - /* Underlying malloc/realloc/free functions */ - void *(*xMalloc)(int); /* underlying malloc(3) function */ - void *(*xRealloc)(void *, int); /* underlying realloc(3) function */ - void (*xFree)(void *); /* underlying free(3) function */ - - /* Mutex to protect pFirst and aHash */ - void (*xEnterMutex)(TmGlobal*); /* Call this to enter the mutex */ - void (*xLeaveMutex)(TmGlobal*); /* Call this to leave mutex */ - void (*xDelMutex)(TmGlobal*); /* Call this to delete mutex */ - void *pMutex; /* Mutex handle */ - - void *(*xSaveMalloc)(void *, size_t); - void *(*xSaveRealloc)(void *, void *, size_t); - void (*xSaveFree)(void *, void *); - - /* OOM injection scheduling. If nCountdown is greater than zero when a - ** malloc attempt is made, it is decremented. If this means nCountdown - ** transitions from 1 to 0, then the allocation fails. If bPersist is true - ** when this happens, nCountdown is then incremented back to 1 (so that the - ** next attempt fails too). - */ - int nCountdown; - int bPersist; - int bEnable; - void (*xHook)(void *); - void *pHookCtx; -}; - -struct TmBlockHdr { - TmBlockHdr *pNext; - TmBlockHdr *pPrev; - int nByte; -#ifdef TM_BACKTRACE - TmAgg *pAgg; -#endif - u32 iForeGuard; -}; - -#ifdef TM_BACKTRACE -struct TmAgg { - int nAlloc; /* Number of allocations at this path */ - int nByte; /* Total number of bytes allocated */ - int nOutAlloc; /* Number of outstanding allocations */ - int nOutByte; /* Number of outstanding bytes */ - void *aFrame[TM_BACKTRACE]; /* backtrace() output */ - TmAgg *pNext; /* Next object in hash-table collision */ -}; -#endif - -#define FOREGUARD 0x80F5E153 -#define REARGUARD 0xE4676B53 -static const u32 rearguard = REARGUARD; - -#define ROUND8(x) (((x)+7)&~7) - -#define BLOCK_HDR_SIZE (ROUND8( sizeof(TmBlockHdr) )) - -static void lsmtest_oom_error(void){ - static int nErr = 0; - nErr++; -} - -static void tmEnterMutex(TmGlobal *pTm){ - pTm->xEnterMutex(pTm); -} -static void tmLeaveMutex(TmGlobal *pTm){ - pTm->xLeaveMutex(pTm); -} - -static void *tmMalloc(TmGlobal *pTm, int nByte){ - TmBlockHdr *pNew; /* New allocation header block */ - u8 *pUser; /* Return value */ - int nReq; /* Total number of bytes requested */ - - assert( sizeof(rearguard)==4 ); - nReq = BLOCK_HDR_SIZE + nByte + 4; - pNew = (TmBlockHdr *)pTm->xMalloc(nReq); - memset(pNew, 0, sizeof(TmBlockHdr)); - - tmEnterMutex(pTm); - assert( pTm->nCountdown>=0 ); - assert( pTm->bPersist==0 || pTm->bPersist==1 ); - - if( pTm->bEnable && pTm->nCountdown==1 ){ - /* Simulate an OOM error. */ - lsmtest_oom_error(); - pTm->xFree(pNew); - pTm->nCountdown = pTm->bPersist; - if( pTm->xHook ) pTm->xHook(pTm->pHookCtx); - pUser = 0; - }else{ - if( pTm->bEnable && pTm->nCountdown ) pTm->nCountdown--; - - pNew->iForeGuard = FOREGUARD; - pNew->nByte = nByte; - pNew->pNext = pTm->pFirst; - - if( pTm->pFirst ){ - pTm->pFirst->pPrev = pNew; - } - pTm->pFirst = pNew; - - pUser = &((u8 *)pNew)[BLOCK_HDR_SIZE]; - memset(pUser, 0x56, nByte); - memcpy(&pUser[nByte], &rearguard, 4); - -#ifdef TM_BACKTRACE - { - TmAgg *pAgg; - int i; - u32 iHash = 0; - void *aFrame[TM_BACKTRACE]; - memset(aFrame, 0, sizeof(aFrame)); - backtrace(aFrame, TM_BACKTRACE); - - for(i=0; i<ArraySize(aFrame); i++){ - iHash += (u64)(aFrame[i]) + (iHash<<3); - } - iHash = iHash % ArraySize(pTm->aHash); - - for(pAgg=pTm->aHash[iHash]; pAgg; pAgg=pAgg->pNext){ - if( memcmp(pAgg->aFrame, aFrame, sizeof(aFrame))==0 ) break; - } - if( !pAgg ){ - pAgg = (TmAgg *)pTm->xMalloc(sizeof(TmAgg)); - memset(pAgg, 0, sizeof(TmAgg)); - memcpy(pAgg->aFrame, aFrame, sizeof(aFrame)); - pAgg->pNext = pTm->aHash[iHash]; - pTm->aHash[iHash] = pAgg; - } - pAgg->nAlloc++; - pAgg->nByte += nByte; - pAgg->nOutAlloc++; - pAgg->nOutByte += nByte; - pNew->pAgg = pAgg; - } -#endif - } - - tmLeaveMutex(pTm); - return pUser; -} - -static void tmFree(TmGlobal *pTm, void *p){ - if( p ){ - TmBlockHdr *pHdr; - u8 *pUser = (u8 *)p; - - tmEnterMutex(pTm); - pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); - assert( pHdr->iForeGuard==FOREGUARD ); - assert( 0==memcmp(&pUser[pHdr->nByte], &rearguard, 4) ); - - if( pHdr->pPrev ){ - assert( pHdr->pPrev->pNext==pHdr ); - pHdr->pPrev->pNext = pHdr->pNext; - }else{ - assert( pHdr==pTm->pFirst ); - pTm->pFirst = pHdr->pNext; - } - if( pHdr->pNext ){ - assert( pHdr->pNext->pPrev==pHdr ); - pHdr->pNext->pPrev = pHdr->pPrev; - } - -#ifdef TM_BACKTRACE - pHdr->pAgg->nOutAlloc--; - pHdr->pAgg->nOutByte -= pHdr->nByte; -#endif - - tmLeaveMutex(pTm); - memset(pUser, 0x58, pHdr->nByte); - memset(pHdr, 0x57, sizeof(TmBlockHdr)); - pTm->xFree(pHdr); - } -} - -static void *tmRealloc(TmGlobal *pTm, void *p, int nByte){ - void *pNew; - - pNew = tmMalloc(pTm, nByte); - if( pNew && p ){ - TmBlockHdr *pHdr; - u8 *pUser = (u8 *)p; - pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); - memcpy(pNew, p, MIN(nByte, pHdr->nByte)); - tmFree(pTm, p); - } - return pNew; -} - -static void tmMallocOom( - TmGlobal *pTm, - int nCountdown, - int bPersist, - void (*xHook)(void *), - void *pHookCtx -){ - assert( nCountdown>=0 ); - assert( bPersist==0 || bPersist==1 ); - pTm->nCountdown = nCountdown; - pTm->bPersist = bPersist; - pTm->xHook = xHook; - pTm->pHookCtx = pHookCtx; - pTm->bEnable = 1; -} - -static void tmMallocOomEnable( - TmGlobal *pTm, - int bEnable -){ - pTm->bEnable = bEnable; -} - -static void tmMallocCheck( - TmGlobal *pTm, - int *pnLeakAlloc, - int *pnLeakByte, - FILE *pFile -){ - TmBlockHdr *pHdr; - int nLeak = 0; - int nByte = 0; - - if( pTm==0 ) return; - - for(pHdr=pTm->pFirst; pHdr; pHdr=pHdr->pNext){ - nLeak++; - nByte += pHdr->nByte; - } - if( pnLeakAlloc ) *pnLeakAlloc = nLeak; - if( pnLeakByte ) *pnLeakByte = nByte; - -#ifdef TM_BACKTRACE - if( pFile ){ - int i; - fprintf(pFile, "LEAKS\n"); - for(i=0; i<ArraySize(pTm->aHash); i++){ - TmAgg *pAgg; - for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ - if( pAgg->nOutAlloc ){ - int j; - fprintf(pFile, "%d %d ", pAgg->nOutByte, pAgg->nOutAlloc); - for(j=0; j<TM_BACKTRACE; j++){ - fprintf(pFile, "%p ", pAgg->aFrame[j]); - } - fprintf(pFile, "\n"); - } - } - } - fprintf(pFile, "\nALLOCATIONS\n"); - for(i=0; i<ArraySize(pTm->aHash); i++){ - TmAgg *pAgg; - for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ - int j; - fprintf(pFile, "%d %d ", pAgg->nByte, pAgg->nAlloc); - for(j=0; j<TM_BACKTRACE; j++) fprintf(pFile, "%p ", pAgg->aFrame[j]); - fprintf(pFile, "\n"); - } - } - } -#else - (void)pFile; -#endif -} - - -#include "lsm.h" -#include "stdlib.h" - -typedef struct LsmMutex LsmMutex; -struct LsmMutex { - lsm_env *pEnv; - lsm_mutex *pMutex; -}; - -static void tmLsmMutexEnter(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)pTm->pMutex; - p->pEnv->xMutexEnter(p->pMutex); -} -static void tmLsmMutexLeave(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)(pTm->pMutex); - p->pEnv->xMutexLeave(p->pMutex); -} -static void tmLsmMutexDel(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)pTm->pMutex; - pTm->xFree(p); -} -static void *tmLsmMalloc(int n){ return malloc(n); } -static void tmLsmFree(void *ptr){ free(ptr); } -static void *tmLsmRealloc(void *ptr, int n){ return realloc(ptr, n); } - -static void *tmLsmEnvMalloc(lsm_env *p, size_t n){ - return tmMalloc((TmGlobal *)(p->pMemCtx), n); -} -static void tmLsmEnvFree(lsm_env *p, void *ptr){ - tmFree((TmGlobal *)(p->pMemCtx), ptr); -} -static void *tmLsmEnvRealloc(lsm_env *p, void *ptr, size_t n){ - return tmRealloc((TmGlobal *)(p->pMemCtx), ptr, n); -} - -void testMallocInstall(lsm_env *pEnv){ - TmGlobal *pGlobal; - LsmMutex *pMutex; - assert( pEnv->pMemCtx==0 ); - - /* Allocate and populate a TmGlobal structure. */ - pGlobal = (TmGlobal *)tmLsmMalloc(sizeof(TmGlobal)); - memset(pGlobal, 0, sizeof(TmGlobal)); - pGlobal->xMalloc = tmLsmMalloc; - pGlobal->xRealloc = tmLsmRealloc; - pGlobal->xFree = tmLsmFree; - pMutex = (LsmMutex *)pGlobal->xMalloc(sizeof(LsmMutex)); - pMutex->pEnv = pEnv; - pEnv->xMutexStatic(pEnv, LSM_MUTEX_HEAP, &pMutex->pMutex); - pGlobal->xEnterMutex = tmLsmMutexEnter; - pGlobal->xLeaveMutex = tmLsmMutexLeave; - pGlobal->xDelMutex = tmLsmMutexDel; - pGlobal->pMutex = (void *)pMutex; - - pGlobal->xSaveMalloc = pEnv->xMalloc; - pGlobal->xSaveRealloc = pEnv->xRealloc; - pGlobal->xSaveFree = pEnv->xFree; - - /* Set up pEnv to the use the new TmGlobal */ - pEnv->pMemCtx = (void *)pGlobal; - pEnv->xMalloc = tmLsmEnvMalloc; - pEnv->xRealloc = tmLsmEnvRealloc; - pEnv->xFree = tmLsmEnvFree; -} - -void testMallocUninstall(lsm_env *pEnv){ - TmGlobal *p = (TmGlobal *)pEnv->pMemCtx; - pEnv->pMemCtx = 0; - if( p ){ - pEnv->xMalloc = p->xSaveMalloc; - pEnv->xRealloc = p->xSaveRealloc; - pEnv->xFree = p->xSaveFree; - p->xDelMutex(p); - tmLsmFree(p); - } -} - -void testMallocCheck( - lsm_env *pEnv, - int *pnLeakAlloc, - int *pnLeakByte, - FILE *pFile -){ - if( pEnv->pMemCtx==0 ){ - *pnLeakAlloc = 0; - *pnLeakByte = 0; - }else{ - tmMallocCheck((TmGlobal *)(pEnv->pMemCtx), pnLeakAlloc, pnLeakByte, pFile); - } -} - -void testMallocOom( - lsm_env *pEnv, - int nCountdown, - int bPersist, - void (*xHook)(void *), - void *pHookCtx -){ - TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); - tmMallocOom(pTm, nCountdown, bPersist, xHook, pHookCtx); -} - -void testMallocOomEnable(lsm_env *pEnv, int bEnable){ - TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); - tmMallocOomEnable(pTm, bEnable); -} diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.c b/ext/lsm1/lsm-test/lsmtest_tdb.c deleted file mode 100644 index 8f63f64ac..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb.c +++ /dev/null @@ -1,846 +0,0 @@ - -/* -** This program attempts to test the correctness of some facets of the -** LSM database library. Specifically, that the contents of the database -** are maintained correctly during a series of inserts and deletes. -*/ - - -#include "lsmtest_tdb.h" -#include "lsm.h" - -#include "lsmtest.h" - -#include <stdlib.h> -#include <string.h> -#include <assert.h> -#ifndef _WIN32 -# include <unistd.h> -#endif -#include <stdio.h> - - -typedef struct SqlDb SqlDb; - -static int error_transaction_function(TestDb *p, int iLevel){ - unused_parameter(p); - unused_parameter(iLevel); - return -1; -} - - -/************************************************************************* -** Begin wrapper for LevelDB. -*/ -#ifdef HAVE_LEVELDB - -#include <leveldb/c.h> - -typedef struct LevelDb LevelDb; -struct LevelDb { - TestDb base; - leveldb_t *db; - leveldb_options_t *pOpt; - leveldb_writeoptions_t *pWriteOpt; - leveldb_readoptions_t *pReadOpt; - - char *pVal; -}; - -static int test_leveldb_close(TestDb *pTestDb){ - LevelDb *pDb = (LevelDb *)pTestDb; - - leveldb_close(pDb->db); - leveldb_writeoptions_destroy(pDb->pWriteOpt); - leveldb_readoptions_destroy(pDb->pReadOpt); - leveldb_options_destroy(pDb->pOpt); - free(pDb->pVal); - free(pDb); - - return 0; -} - -static int test_leveldb_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - leveldb_put(pDb->db, pDb->pWriteOpt, pKey, nKey, pVal, nVal, &zErr); - return (zErr!=0); -} - -static int test_leveldb_delete(TestDb *pTestDb, void *pKey, int nKey){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - leveldb_delete(pDb->db, pDb->pWriteOpt, pKey, nKey, &zErr); - return (zErr!=0); -} - -static int test_leveldb_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - size_t nVal = 0; - - if( pKey==0 ) return 0; - free(pDb->pVal); - pDb->pVal = leveldb_get(pDb->db, pDb->pReadOpt, pKey, nKey, &nVal, &zErr); - *ppVal = (void *)(pDb->pVal); - if( pDb->pVal==0 ){ - *pnVal = -1; - }else{ - *pnVal = (int)nVal; - } - - return (zErr!=0); -} - -static int test_leveldb_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *, void *, int , void *, int) -){ - LevelDb *pDb = (LevelDb *)pTestDb; - leveldb_iterator_t *iter; - - iter = leveldb_create_iterator(pDb->db, pDb->pReadOpt); - - if( bReverse==0 ){ - if( pKey1 ){ - leveldb_iter_seek(iter, pKey1, nKey1); - }else{ - leveldb_iter_seek_to_first(iter); - } - }else{ - if( pKey2 ){ - leveldb_iter_seek(iter, pKey2, nKey2); - - if( leveldb_iter_valid(iter)==0 ){ - leveldb_iter_seek_to_last(iter); - }else{ - const char *k; size_t n; - int res; - k = leveldb_iter_key(iter, &n); - res = memcmp(k, pKey2, MIN(n, nKey2)); - if( res==0 ) res = n - nKey2; - assert( res>=0 ); - if( res>0 ){ - leveldb_iter_prev(iter); - } - } - }else{ - leveldb_iter_seek_to_last(iter); - } - } - - - while( leveldb_iter_valid(iter) ){ - const char *k; size_t n; - const char *v; size_t n2; - int res; - - k = leveldb_iter_key(iter, &n); - if( bReverse==0 && pKey2 ){ - res = memcmp(k, pKey2, MIN(n, nKey2)); - if( res==0 ) res = n - nKey2; - if( res>0 ) break; - } - if( bReverse!=0 && pKey1 ){ - res = memcmp(k, pKey1, MIN(n, nKey1)); - if( res==0 ) res = n - nKey1; - if( res<0 ) break; - } - - v = leveldb_iter_value(iter, &n2); - - xCallback(pCtx, (void *)k, n, (void *)v, n2); - - if( bReverse==0 ){ - leveldb_iter_next(iter); - }else{ - leveldb_iter_prev(iter); - } - } - - leveldb_iter_destroy(iter); - return 0; -} - -static int test_leveldb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods LeveldbMethods = { - test_leveldb_close, - test_leveldb_write, - test_leveldb_delete, - 0, - test_leveldb_fetch, - test_leveldb_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - LevelDb *pLevelDb; - char *zErr = 0; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pLevelDb = (LevelDb *)malloc(sizeof(LevelDb)); - memset(pLevelDb, 0, sizeof(LevelDb)); - - pLevelDb->pOpt = leveldb_options_create(); - leveldb_options_set_create_if_missing(pLevelDb->pOpt, 1); - pLevelDb->pWriteOpt = leveldb_writeoptions_create(); - pLevelDb->pReadOpt = leveldb_readoptions_create(); - - pLevelDb->db = leveldb_open(pLevelDb->pOpt, zFilename, &zErr); - - if( zErr ){ - test_leveldb_close((TestDb *)pLevelDb); - *ppDb = 0; - return 1; - } - - *ppDb = (TestDb *)pLevelDb; - pLevelDb->base.pMethods = &LeveldbMethods; - return 0; -} -#endif /* HAVE_LEVELDB */ -/* -** End wrapper for LevelDB. -*************************************************************************/ - -#ifdef HAVE_KYOTOCABINET -static int kc_close(TestDb *pTestDb){ - return test_kc_close(pTestDb); -} - -static int kc_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - return test_kc_write(pTestDb, pKey, nKey, pVal, nVal); -} - -static int kc_delete(TestDb *pTestDb, void *pKey, int nKey){ - return test_kc_delete(pTestDb, pKey, nKey); -} - -static int kc_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - return test_kc_delete_range(pTestDb, pKey1, nKey1, pKey2, nKey2); -} - -static int kc_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - if( pKey==0 ) return LSM_OK; - return test_kc_fetch(pTestDb, pKey, nKey, ppVal, pnVal); -} - -static int kc_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - return test_kc_scan( - pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback - ); -} - -static int kc_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods KcdbMethods = { - kc_close, - kc_write, - kc_delete, - kc_delete_range, - kc_fetch, - kc_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - int rc; - TestDb *pTestDb = 0; - - rc = test_kc_open(zFilename, bClear, &pTestDb); - if( rc!=0 ){ - *ppDb = 0; - return rc; - } - pTestDb->pMethods = &KcdbMethods; - *ppDb = pTestDb; - return 0; -} -#endif /* HAVE_KYOTOCABINET */ -/* -** End wrapper for Kyoto cabinet. -*************************************************************************/ - -#ifdef HAVE_MDB -static int mdb_close(TestDb *pTestDb){ - return test_mdb_close(pTestDb); -} - -static int mdb_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - return test_mdb_write(pTestDb, pKey, nKey, pVal, nVal); -} - -static int mdb_delete(TestDb *pTestDb, void *pKey, int nKey){ - return test_mdb_delete(pTestDb, pKey, nKey); -} - -static int mdb_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - if( pKey==0 ) return LSM_OK; - return test_mdb_fetch(pTestDb, pKey, nKey, ppVal, pnVal); -} - -static int mdb_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - return test_mdb_scan( - pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback - ); -} - -static int mdb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods KcdbMethods = { - mdb_close, - mdb_write, - mdb_delete, - 0, - mdb_fetch, - mdb_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - int rc; - TestDb *pTestDb = 0; - - rc = test_mdb_open(zSpec, zFilename, bClear, &pTestDb); - if( rc!=0 ){ - *ppDb = 0; - return rc; - } - pTestDb->pMethods = &KcdbMethods; - *ppDb = pTestDb; - return 0; -} -#endif /* HAVE_MDB */ - -/************************************************************************* -** Begin wrapper for SQLite. -*/ - -/* -** nOpenTrans: -** The number of open nested transactions, in the same sense as used -** by the tdb_begin/commit/rollback and SQLite 4 KV interfaces. If this -** value is 0, there are no transactions open at all. If it is 1, then -** there is a read transaction. If it is 2 or greater, then there are -** (nOpenTrans-1) nested write transactions open. -*/ -struct SqlDb { - TestDb base; - sqlite3 *db; - sqlite3_stmt *pInsert; - sqlite3_stmt *pDelete; - sqlite3_stmt *pDeleteRange; - sqlite3_stmt *pFetch; - sqlite3_stmt *apScan[8]; - - int nOpenTrans; - - /* Used by sql_fetch() to allocate space for results */ - int nAlloc; - u8 *aAlloc; -}; - -static int sql_close(TestDb *pTestDb){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_finalize(pDb->pInsert); - sqlite3_finalize(pDb->pDelete); - sqlite3_finalize(pDb->pDeleteRange); - sqlite3_finalize(pDb->pFetch); - sqlite3_finalize(pDb->apScan[0]); - sqlite3_finalize(pDb->apScan[1]); - sqlite3_finalize(pDb->apScan[2]); - sqlite3_finalize(pDb->apScan[3]); - sqlite3_finalize(pDb->apScan[4]); - sqlite3_finalize(pDb->apScan[5]); - sqlite3_finalize(pDb->apScan[6]); - sqlite3_finalize(pDb->apScan[7]); - sqlite3_close(pDb->db); - free((char *)pDb->aAlloc); - free((char *)pDb); - return SQLITE_OK; -} - -static int sql_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pInsert, 1, pKey, nKey, SQLITE_STATIC); - sqlite3_bind_blob(pDb->pInsert, 2, pVal, nVal, SQLITE_STATIC); - sqlite3_step(pDb->pInsert); - return sqlite3_reset(pDb->pInsert); -} - -static int sql_delete(TestDb *pTestDb, void *pKey, int nKey){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pDelete, 1, pKey, nKey, SQLITE_STATIC); - sqlite3_step(pDb->pDelete); - return sqlite3_reset(pDb->pDelete); -} - -static int sql_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pDeleteRange, 1, pKey1, nKey1, SQLITE_STATIC); - sqlite3_bind_blob(pDb->pDeleteRange, 2, pKey2, nKey2, SQLITE_STATIC); - sqlite3_step(pDb->pDeleteRange); - return sqlite3_reset(pDb->pDeleteRange); -} - -static int sql_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - SqlDb *pDb = (SqlDb *)pTestDb; - int rc; - - sqlite3_reset(pDb->pFetch); - if( pKey==0 ){ - assert( ppVal==0 ); - assert( pnVal==0 ); - return LSM_OK; - } - - sqlite3_bind_blob(pDb->pFetch, 1, pKey, nKey, SQLITE_STATIC); - rc = sqlite3_step(pDb->pFetch); - if( rc==SQLITE_ROW ){ - int nVal = sqlite3_column_bytes(pDb->pFetch, 0); - u8 *aVal = (void *)sqlite3_column_blob(pDb->pFetch, 0); - - if( nVal>pDb->nAlloc ){ - free(pDb->aAlloc); - pDb->aAlloc = (u8 *)malloc(nVal*2); - pDb->nAlloc = nVal*2; - } - memcpy(pDb->aAlloc, aVal, nVal); - *pnVal = nVal; - *ppVal = (void *)pDb->aAlloc; - }else{ - *pnVal = -1; - *ppVal = 0; - } - - rc = sqlite3_reset(pDb->pFetch); - return rc; -} - -static int sql_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_stmt *pScan; - - assert( bReverse==1 || bReverse==0 ); - pScan = pDb->apScan[(pFirst==0) + (pLast==0)*2 + bReverse*4]; - - if( pFirst ) sqlite3_bind_blob(pScan, 1, pFirst, nFirst, SQLITE_STATIC); - if( pLast ) sqlite3_bind_blob(pScan, 2, pLast, nLast, SQLITE_STATIC); - - while( SQLITE_ROW==sqlite3_step(pScan) ){ - void *pKey; int nKey; - void *pVal; int nVal; - - nKey = sqlite3_column_bytes(pScan, 0); - pKey = (void *)sqlite3_column_blob(pScan, 0); - nVal = sqlite3_column_bytes(pScan, 1); - pVal = (void *)sqlite3_column_blob(pScan, 1); - - xCallback(pCtx, pKey, nKey, pVal, nVal); - } - return sqlite3_reset(pScan); -} - -static int sql_begin(TestDb *pTestDb, int iLevel){ - int i; - SqlDb *pDb = (SqlDb *)pTestDb; - - /* iLevel==0 is a no-op */ - if( iLevel==0 ) return 0; - - /* If there are no transactions at all open, open a read transaction. */ - if( pDb->nOpenTrans==0 ){ - int rc = sqlite3_exec(pDb->db, - "BEGIN; SELECT * FROM sqlite_schema LIMIT 1;" , 0, 0, 0 - ); - if( rc!=0 ) return rc; - pDb->nOpenTrans = 1; - } - - /* Open any required write transactions */ - for(i=pDb->nOpenTrans; i<iLevel; i++){ - char *zSql = sqlite3_mprintf("SAVEPOINT x%d", i); - int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=SQLITE_OK ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_commit(TestDb *pTestDb, int iLevel){ - SqlDb *pDb = (SqlDb *)pTestDb; - assert( iLevel>=0 ); - - /* Close the read transaction if requested. */ - if( pDb->nOpenTrans>=1 && iLevel==0 ){ - int rc = sqlite3_exec(pDb->db, "COMMIT", 0, 0, 0); - if( rc!=0 ) return rc; - pDb->nOpenTrans = 0; - } - - /* Close write transactions as required */ - if( pDb->nOpenTrans>iLevel ){ - char *zSql = sqlite3_mprintf("RELEASE x%d", iLevel); - int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=0 ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_rollback(TestDb *pTestDb, int iLevel){ - SqlDb *pDb = (SqlDb *)pTestDb; - assert( iLevel>=0 ); - - if( pDb->nOpenTrans>=1 && iLevel==0 ){ - /* Close the read transaction if requested. */ - int rc = sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0); - if( rc!=0 ) return rc; - }else if( pDb->nOpenTrans>1 && iLevel==1 ){ - /* Or, rollback and close the top-level write transaction */ - int rc = sqlite3_exec(pDb->db, "ROLLBACK TO x1; RELEASE x1;", 0, 0, 0); - if( rc!=0 ) return rc; - }else{ - /* Or, just roll back some nested transactions */ - char *zSql = sqlite3_mprintf("ROLLBACK TO x%d", iLevel-1); - int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=0 ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods SqlMethods = { - sql_close, - sql_write, - sql_delete, - sql_delete_range, - sql_fetch, - sql_scan, - sql_begin, - sql_commit, - sql_rollback - }; - const char *zCreate = "CREATE TABLE IF NOT EXISTS t1(k PRIMARY KEY, v)"; - const char *zInsert = "REPLACE INTO t1 VALUES(?, ?)"; - const char *zDelete = "DELETE FROM t1 WHERE k = ?"; - const char *zRange = "DELETE FROM t1 WHERE k>? AND k<?"; - const char *zFetch = "SELECT v FROM t1 WHERE k = ?"; - - const char *zScan0 = "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k"; - const char *zScan1 = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k"; - const char *zScan2 = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k"; - const char *zScan3 = "SELECT * FROM t1 ORDER BY k"; - - const char *zScan4 = - "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k DESC"; - const char *zScan5 = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k DESC"; - const char *zScan6 = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k DESC"; - const char *zScan7 = "SELECT * FROM t1 ORDER BY k DESC"; - - int rc; - SqlDb *pDb; - char *zPragma; - - if( bClear && zFilename && zFilename[0] ){ - unlink(zFilename); - } - - pDb = (SqlDb *)malloc(sizeof(SqlDb)); - memset(pDb, 0, sizeof(SqlDb)); - pDb->base.pMethods = &SqlMethods; - - if( 0!=(rc = sqlite3_open(zFilename, &pDb->db)) - || 0!=(rc = sqlite3_exec(pDb->db, zCreate, 0, 0, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zInsert, -1, &pDb->pInsert, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zDelete, -1, &pDb->pDelete, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zRange, -1, &pDb->pDeleteRange, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zFetch, -1, &pDb->pFetch, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan0, -1, &pDb->apScan[0], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan1, -1, &pDb->apScan[1], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan2, -1, &pDb->apScan[2], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan3, -1, &pDb->apScan[3], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan4, -1, &pDb->apScan[4], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan5, -1, &pDb->apScan[5], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan6, -1, &pDb->apScan[6], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan7, -1, &pDb->apScan[7], 0)) - ){ - *ppDb = 0; - sql_close((TestDb *)pDb); - return rc; - } - - zPragma = sqlite3_mprintf("PRAGMA page_size=%d", TESTDB_DEFAULT_PAGE_SIZE); - sqlite3_exec(pDb->db, zPragma, 0, 0, 0); - sqlite3_free(zPragma); - zPragma = sqlite3_mprintf("PRAGMA cache_size=%d", TESTDB_DEFAULT_CACHE_SIZE); - sqlite3_exec(pDb->db, zPragma, 0, 0, 0); - sqlite3_free(zPragma); - - /* sqlite3_exec(pDb->db, "PRAGMA locking_mode=EXCLUSIVE", 0, 0, 0); */ - sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); - sqlite3_exec(pDb->db, "PRAGMA journal_mode=WAL", 0, 0, 0); - sqlite3_exec(pDb->db, "PRAGMA wal_autocheckpoint=4096", 0, 0, 0); - if( zSpec ){ - rc = sqlite3_exec(pDb->db, zSpec, 0, 0, 0); - if( rc!=SQLITE_OK ){ - sql_close((TestDb *)pDb); - return rc; - } - } - - *ppDb = (TestDb *)pDb; - return 0; -} -/* -** End wrapper for SQLite. -*************************************************************************/ - -/************************************************************************* -** Begin exported functions. -*/ -static struct Lib { - const char *zName; - const char *zDefaultDb; - int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb); -} aLib[] = { - { "sqlite3", "testdb.sqlite", sql_open }, - { "lsm_small", "testdb.lsm_small", test_lsm_small_open }, - { "lsm_lomem", "testdb.lsm_lomem", test_lsm_lomem_open }, - { "lsm_lomem2", "testdb.lsm_lomem2", test_lsm_lomem2_open }, -#ifdef HAVE_ZLIB - { "lsm_zip", "testdb.lsm_zip", test_lsm_zip_open }, -#endif - { "lsm", "testdb.lsm", test_lsm_open }, -#ifdef LSM_MUTEX_PTHREADS - { "lsm_mt2", "testdb.lsm_mt2", test_lsm_mt2 }, - { "lsm_mt3", "testdb.lsm_mt3", test_lsm_mt3 }, -#endif -#ifdef HAVE_LEVELDB - { "leveldb", "testdb.leveldb", test_leveldb_open }, -#endif -#ifdef HAVE_KYOTOCABINET - { "kyotocabinet", "testdb.kc", kc_open }, -#endif -#ifdef HAVE_MDB - { "mdb", "./testdb.mdb", mdb_open } -#endif -}; - -const char *tdb_system_name(int i){ - if( i<0 || i>=ArraySize(aLib) ) return 0; - return aLib[i].zName; -} - -const char *tdb_default_db(const char *zSys){ - int i; - for(i=0; i<ArraySize(aLib); i++){ - if( strcmp(aLib[i].zName, zSys)==0 ) return aLib[i].zDefaultDb; - } - return 0; -} - -int tdb_open(const char *zLib, const char *zDb, int bClear, TestDb **ppDb){ - int i; - int rc = 1; - const char *zSpec = 0; - - int nLib = 0; - while( zLib[nLib] && zLib[nLib]!=' ' ){ - nLib++; - } - zSpec = &zLib[nLib]; - while( *zSpec==' ' ) zSpec++; - if( *zSpec=='\0' ) zSpec = 0; - - for(i=0; i<ArraySize(aLib); i++){ - if( (int)strlen(aLib[i].zName)==nLib - && 0==memcmp(zLib, aLib[i].zName, nLib) ){ - rc = aLib[i].xOpen(zSpec, (zDb ? zDb : aLib[i].zDefaultDb), bClear, ppDb); - if( rc==0 ){ - (*ppDb)->zLibrary = aLib[i].zName; - } - break; - } - } - - if( rc ){ - /* Failed to find the requested database library. Return an error. */ - *ppDb = 0; - } - return rc; -} - -int tdb_close(TestDb *pDb){ - if( pDb ){ - return pDb->pMethods->xClose(pDb); - } - return 0; -} - -int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - return pDb->pMethods->xWrite(pDb, pKey, nKey, pVal, nVal); -} - -int tdb_delete(TestDb *pDb, void *pKey, int nKey){ - return pDb->pMethods->xDelete(pDb, pKey, nKey); -} - -int tdb_delete_range( - TestDb *pDb, void *pKey1, int nKey1, void *pKey2, int nKey2 -){ - return pDb->pMethods->xDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2); -} - -int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal){ - return pDb->pMethods->xFetch(pDb, pKey, nKey, ppVal, pnVal); -} - -int tdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True to scan in reverse order */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - return pDb->pMethods->xScan( - pDb, pCtx, bReverse, pKey1, nKey1, pKey2, nKey2, xCallback - ); -} - -int tdb_begin(TestDb *pDb, int iLevel){ - return pDb->pMethods->xBegin(pDb, iLevel); -} -int tdb_commit(TestDb *pDb, int iLevel){ - return pDb->pMethods->xCommit(pDb, iLevel); -} -int tdb_rollback(TestDb *pDb, int iLevel){ - return pDb->pMethods->xRollback(pDb, iLevel); -} - -int tdb_transaction_support(TestDb *pDb){ - return (pDb->pMethods->xBegin != error_transaction_function); -} - -const char *tdb_library_name(TestDb *pDb){ - return pDb->zLibrary; -} - -/* -** End exported functions. -*************************************************************************/ diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.h b/ext/lsm1/lsm-test/lsmtest_tdb.h deleted file mode 100644 index c55b6e2f8..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb.h +++ /dev/null @@ -1,174 +0,0 @@ - -/* -** This file is the interface to a very simple database library used for -** testing. The interface is similar to that of the LSM. The main virtue -** of this library is that the same API may be used to access a key-value -** store implemented by LSM, SQLite or another database system. Which -** makes it easy to use for correctness and performance tests. -*/ - -#ifndef __WRAPPER_H_ -#define __WRAPPER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "lsm.h" - -typedef struct TestDb TestDb; - -/* -** Open a new database connection. The first argument is the name of the -** database library to use. e.g. something like: -** -** "sqlite3" -** "lsm" -** -** See function tdb_system_name() for a list of available database systems. -** -** The second argument is the name of the database to open (e.g. a filename). -** -** If the third parameter is non-zero, then any existing database by the -** name of zDb is removed before opening a new one. If it is zero, then an -** existing database may be opened. -*/ -int tdb_open(const char *zLibrary, const char *zDb, int bClear, TestDb **ppDb); - -/* -** Close a database handle. -*/ -int tdb_close(TestDb *pDb); - -/* -** Write a new key/value into the database. -*/ -int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal); - -/* -** Delete a key from the database. -*/ -int tdb_delete(TestDb *pDb, void *pKey, int nKey); - -/* -** Delete a range of keys from the database. -*/ -int tdb_delete_range(TestDb *, void *pKey1, int nKey1, void *pKey2, int nKey2); - -/* -** Query the database for key (pKey/nKey). If no entry is found, set *ppVal -** to 0 and *pnVal to -1 before returning. Otherwise, set *ppVal and *pnVal -** to a pointer to and size of the value associated with (pKey/nKey). -*/ -int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal); - -/* -** Open and close nested transactions. Currently, these functions only -** work for SQLite3 and LSM systems. Use the tdb_transaction_support() -** function to determine if a given TestDb handle supports these methods. -** -** These functions and the iLevel parameter follow the same conventions as -** the SQLite 4 transaction interface. Note that this is slightly different -** from the way LSM does things. As follows: -** -** tdb_begin(): -** A successful call to tdb_begin() with (iLevel>1) guarantees that -** there are at least (iLevel-1) write transactions open. If iLevel==1, -** then it guarantees that at least a read-transaction is open. Calling -** tdb_begin() with iLevel==0 is a no-op. -** -** tdb_commit(): -** A successful call to tdb_commit() with (iLevel>1) guarantees that -** there are at most (iLevel-1) write transactions open. If iLevel==1, -** then it guarantees that there are no write transactions open (although -** a read-transaction may remain open). Calling tdb_commit() with -** iLevel==0 ensures that all transactions, read or write, have been -** closed and committed. -** -** tdb_rollback(): -** This call is similar to tdb_commit(), except that instead of committing -** transactions, it reverts them. For example, calling tdb_rollback() with -** iLevel==2 ensures that there is at most one write transaction open, and -** restores the database to the state that it was in when that transaction -** was opened. -** -** In other words, tdb_commit() just closes transactions - tdb_rollback() -** closes transactions and then restores the database to the state it -** was in before those transactions were even opened. -*/ -int tdb_begin(TestDb *pDb, int iLevel); -int tdb_commit(TestDb *pDb, int iLevel); -int tdb_rollback(TestDb *pDb, int iLevel); - -/* -** Return true if transactions are supported, or false otherwise. -*/ -int tdb_transaction_support(TestDb *pDb); - -/* -** Return the name of the database library (as passed to tdb_open()) used -** by the handled passed as the first argument. -*/ -const char *tdb_library_name(TestDb *pDb); - -/* -** Scan a range of database keys. Invoke the callback function for each -** key visited. -*/ -int tdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True to scan in reverse order */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -); - -const char *tdb_system_name(int i); -const char *tdb_default_db(const char *zSys); - -int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb); - -/* -** If the TestDb handle passed as an argument is a wrapper around an LSM -** database, return the LSM handle. Otherwise, if the argument is some other -** database system, return NULL. -*/ -lsm_db *tdb_lsm(TestDb *pDb); - -/* -** Return true if the db passed as an argument is a multi-threaded LSM -** connection. -*/ -int tdb_lsm_multithread(TestDb *pDb); - -/* -** Return a pointer to the lsm_env object used by all lsm database -** connections initialized as a copy of the object returned by -** lsm_default_env(). It may be modified (e.g. to override functions) -** if the caller can guarantee that it is not already in use. -*/ -lsm_env *tdb_lsm_env(void); - -/* -** The following functions only work with LSM database handles. It is -** illegal to call them with any other type of database handle specified -** as an argument. -*/ -void tdb_lsm_enable_log(TestDb *pDb, int bEnable); -void tdb_lsm_application_crash(TestDb *pDb); -void tdb_lsm_prepare_system_crash(TestDb *pDb); -void tdb_lsm_system_crash(TestDb *pDb); -void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync); - - -void tdb_lsm_safety(TestDb *pDb, int eMode); -void tdb_lsm_config_work_hook(TestDb *pDb, void (*)(lsm_db *, void *), void *); -void tdb_lsm_write_hook(TestDb *, void(*)(void*,int,lsm_i64,int,int), void*); -int tdb_lsm_config_str(TestDb *pDb, const char *zStr); - -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif - -#endif diff --git a/ext/lsm1/lsm-test/lsmtest_tdb2.cc b/ext/lsm1/lsm-test/lsmtest_tdb2.cc deleted file mode 100644 index 86ebb4958..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb2.cc +++ /dev/null @@ -1,369 +0,0 @@ - - -#include "lsmtest.h" -#include <stdlib.h> - -#ifdef HAVE_KYOTOCABINET -#include "kcpolydb.h" -extern "C" { - struct KcDb { - TestDb base; - kyotocabinet::TreeDB* db; - char *pVal; - }; -} - -int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb){ - KcDb *pKcDb; - int ok; - int rc = 0; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pKcDb = (KcDb *)malloc(sizeof(KcDb)); - memset(pKcDb, 0, sizeof(KcDb)); - - - pKcDb->db = new kyotocabinet::TreeDB(); - pKcDb->db->tune_page(TESTDB_DEFAULT_PAGE_SIZE); - pKcDb->db->tune_page_cache( - TESTDB_DEFAULT_PAGE_SIZE * TESTDB_DEFAULT_CACHE_SIZE - ); - ok = pKcDb->db->open(zFilename, - kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE - ); - if( ok==0 ){ - free(pKcDb); - pKcDb = 0; - rc = 1; - } - - *ppDb = (TestDb *)pKcDb; - return rc; -} - -int test_kc_close(TestDb *pDb){ - KcDb *pKcDb = (KcDb *)pDb; - if( pKcDb->pVal ){ - delete [] pKcDb->pVal; - } - pKcDb->db->close(); - delete pKcDb->db; - free(pKcDb); - return 0; -} - -int test_kc_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - KcDb *pKcDb = (KcDb *)pDb; - int ok; - - ok = pKcDb->db->set((const char *)pKey, nKey, (const char *)pVal, nVal); - return (ok ? 0 : 1); -} - -int test_kc_delete(TestDb *pDb, void *pKey, int nKey){ - KcDb *pKcDb = (KcDb *)pDb; - int ok; - - ok = pKcDb->db->remove((const char *)pKey, nKey); - return (ok ? 0 : 1); -} - -int test_kc_delete_range( - TestDb *pDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - int res; - KcDb *pKcDb = (KcDb *)pDb; - kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); - - if( pKey1 ){ - res = pCur->jump((const char *)pKey1, nKey1); - }else{ - res = pCur->jump(); - } - - while( 1 ){ - const char *pKey; size_t nKey; - const char *pVal; size_t nVal; - - pKey = pCur->get(&nKey, &pVal, &nVal); - if( pKey==0 ) break; - -#ifndef NDEBUG - if( pKey1 ){ - res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); - assert( res>0 || (res==0 && nKey>nKey1) ); - } -#endif - - if( pKey2 ){ - res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); - if( res>0 || (res==0 && (size_t)nKey2<nKey) ){ - delete [] pKey; - break; - } - } - pCur->remove(); - delete [] pKey; - } - - delete pCur; - return 0; -} - -int test_kc_fetch( - TestDb *pDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - KcDb *pKcDb = (KcDb *)pDb; - size_t nVal; - - if( pKcDb->pVal ){ - delete [] pKcDb->pVal; - pKcDb->pVal = 0; - } - - pKcDb->pVal = pKcDb->db->get((const char *)pKey, nKey, &nVal); - if( pKcDb->pVal ){ - *ppVal = pKcDb->pVal; - *pnVal = nVal; - }else{ - *ppVal = 0; - *pnVal = -1; - } - - return 0; -} - -int test_kc_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True for a reverse order scan */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - KcDb *pKcDb = (KcDb *)pDb; - kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); - int res; - - if( bReverse==0 ){ - if( pKey1 ){ - res = pCur->jump((const char *)pKey1, nKey1); - }else{ - res = pCur->jump(); - } - }else{ - if( pKey2 ){ - res = pCur->jump_back((const char *)pKey2, nKey2); - }else{ - res = pCur->jump_back(); - } - } - - while( res ){ - const char *pKey; size_t nKey; - const char *pVal; size_t nVal; - pKey = pCur->get(&nKey, &pVal, &nVal); - - if( bReverse==0 && pKey2 ){ - res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); - if( res>0 || (res==0 && (size_t)nKey2<nKey) ){ - delete [] pKey; - break; - } - }else if( bReverse!=0 && pKey1 ){ - res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); - if( res<0 || (res==0 && (size_t)nKey1>nKey) ){ - delete [] pKey; - break; - } - } - - xCallback(pCtx, (void *)pKey, (int)nKey, (void *)pVal, (int)nVal); - delete [] pKey; - - if( bReverse ){ - res = pCur->step_back(); - }else{ - res = pCur->step(); - } - } - - delete pCur; - return 0; -} -#endif /* HAVE_KYOTOCABINET */ - -#ifdef HAVE_MDB -#include "lmdb.h" - -extern "C" { - struct MdbDb { - TestDb base; - MDB_env *env; - MDB_dbi dbi; - }; -} - -int test_mdb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - MDB_txn *txn; - MdbDb *pMdb; - int rc; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pMdb = (MdbDb *)malloc(sizeof(MdbDb)); - memset(pMdb, 0, sizeof(MdbDb)); - - rc = mdb_env_create(&pMdb->env); - if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024); - if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600); - if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_open(txn, NULL, 0, &pMdb->dbi); - mdb_txn_commit(txn); - } - - *ppDb = (TestDb *)pMdb; - return rc; -} - -int test_mdb_close(TestDb *pDb){ - MdbDb *pMdb = (MdbDb *)pDb; - - mdb_close(pMdb->env, pMdb->dbi); - mdb_env_close(pMdb->env); - free(pMdb); - return 0; -} - -int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val val; - MDB_val key; - MDB_txn *txn; - - val.mv_size = nVal; - val.mv_data = pVal; - key.mv_size = nKey; - key.mv_data = pKey; - - rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_put(txn, pMdb->dbi, &key, &val, 0); - if( rc==0 ){ - rc = mdb_txn_commit(txn); - }else{ - mdb_txn_abort(txn); - } - } - - return rc; -} - -int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val key; - MDB_txn *txn; - - key.mv_size = nKey; - key.mv_data = pKey; - rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_del(txn, pMdb->dbi, &key, 0); - if( rc==0 ){ - rc = mdb_txn_commit(txn); - }else{ - mdb_txn_abort(txn); - } - } - - return rc; -} - -int test_mdb_fetch( - TestDb *pDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val key; - MDB_txn *txn; - - key.mv_size = nKey; - key.mv_data = pKey; - - rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); - if( rc==0 ){ - MDB_val val = {0, 0}; - rc = mdb_get(txn, pMdb->dbi, &key, &val); - if( rc==MDB_NOTFOUND ){ - rc = 0; - *ppVal = 0; - *pnVal = -1; - }else{ - *ppVal = val.mv_data; - *pnVal = val.mv_size; - } - mdb_txn_commit(txn); - } - - return rc; -} - -int test_mdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True for a reverse order scan */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - MdbDb *pMdb = (MdbDb *)pDb; - int rc; - MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT; - MDB_txn *txn; - - rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); - if( rc==0 ){ - MDB_cursor *csr; - MDB_val key = {0, 0}; - MDB_val val = {0, 0}; - - rc = mdb_cursor_open(txn, pMdb->dbi, &csr); - if( rc==0 ){ - while( mdb_cursor_get(csr, &key, &val, op)==0 ){ - xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size); - } - mdb_cursor_close(csr); - } - } - - return rc; -} - -#endif /* HAVE_MDB */ diff --git a/ext/lsm1/lsm-test/lsmtest_tdb3.c b/ext/lsm1/lsm-test/lsmtest_tdb3.c deleted file mode 100644 index e29497af2..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb3.c +++ /dev/null @@ -1,1429 +0,0 @@ - -#include "lsmtest_tdb.h" -#include "lsm.h" -#include "lsmtest.h" - -#include <stdlib.h> -#include <string.h> -#include <assert.h> -#ifndef _WIN32 -# include <unistd.h> -#endif -#include <stdio.h> - -#ifndef _WIN32 -# include <sys/time.h> -#endif - -typedef struct LsmDb LsmDb; -typedef struct LsmWorker LsmWorker; -typedef struct LsmFile LsmFile; - -#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024) -#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024) - -#ifdef LSM_MUTEX_PTHREADS -#include <pthread.h> - -#define LSMTEST_THREAD_CKPT 1 -#define LSMTEST_THREAD_WORKER 2 -#define LSMTEST_THREAD_WORKER_AC 3 - -/* -** There are several different types of worker threads that run in different -** test configurations, depending on the value of LsmWorker.eType. -** -** 1. Checkpointer. -** 2. Worker with auto-checkpoint. -** 3. Worker without auto-checkpoint. -*/ -struct LsmWorker { - LsmDb *pDb; /* Main database structure */ - lsm_db *pWorker; /* Worker database handle */ - pthread_t worker_thread; /* Worker thread */ - pthread_cond_t worker_cond; /* Condition var the worker waits on */ - pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ - int bDoWork; /* Set to true by client when there is work */ - int worker_rc; /* Store error code here */ - int eType; /* LSMTEST_THREAD_XXX constant */ - int bBlock; -}; -#else -struct LsmWorker { int worker_rc; int bBlock; }; -#endif - -static void mt_shutdown(LsmDb *); - -lsm_env *tdb_lsm_env(void){ - static int bInit = 0; - static lsm_env env; - if( bInit==0 ){ - memcpy(&env, lsm_default_env(), sizeof(env)); - bInit = 1; - } - return &env; -} - -typedef struct FileSector FileSector; -typedef struct FileData FileData; - -struct FileSector { - u8 *aOld; /* Old data for this sector */ -}; - -struct FileData { - int nSector; /* Allocated size of apSector[] array */ - FileSector *aSector; /* Array of file sectors */ -}; - -/* -** bPrepareCrash: -** If non-zero, the file wrappers maintain enough in-memory data to -** simulate the effect of a power-failure on the file-system (i.e. that -** unsynced sectors may be written, not written, or overwritten with -** arbitrary data when the crash occurs). -** -** bCrashed: -** Set to true after a crash is simulated. Once this variable is true, all -** VFS methods other than xClose() return LSM_IOERR as soon as they are -** called (without affecting the contents of the file-system). -** -** env: -** The environment object used by all lsm_db* handles opened by this -** object (i.e. LsmDb.db plus any worker connections). Variable env.pVfsCtx -** always points to the containing LsmDb structure. -*/ -struct LsmDb { - TestDb base; /* Base class - methods table */ - lsm_env env; /* Environment used by connection db */ - char *zName; /* Database file name */ - lsm_db *db; /* LSM database handle */ - - lsm_cursor *pCsr; /* Cursor held open during read transaction */ - void *pBuf; /* Buffer for tdb_fetch() output */ - int nBuf; /* Allocated (not used) size of pBuf */ - - /* Crash testing related state */ - int bCrashed; /* True once a crash has occurred */ - int nAutoCrash; /* Number of syncs until a crash */ - int bPrepareCrash; /* True to store writes in memory */ - - /* Unsynced data (while crash testing) */ - int szSector; /* Assumed size of disk sectors (512B) */ - FileData aFile[2]; /* Database and log file data */ - - /* Other test instrumentation */ - int bNoRecovery; /* If true, assume DMS2 is locked */ - - /* Work hook redirection */ - void (*xWork)(lsm_db *, void *); - void *pWorkCtx; - - /* IO logging hook */ - void (*xWriteHook)(void *, int, lsm_i64, int, int); - void *pWriteCtx; - - /* Worker threads (for lsm_mt) */ - int nMtMinCkpt; - int nMtMaxCkpt; - int eMode; - int nWorker; - LsmWorker *aWorker; -}; - -#define LSMTEST_MODE_SINGLETHREAD 1 -#define LSMTEST_MODE_BACKGROUND_CKPT 2 -#define LSMTEST_MODE_BACKGROUND_WORK 3 -#define LSMTEST_MODE_BACKGROUND_BOTH 4 - -/************************************************************************* -************************************************************************** -** Begin test VFS code. -*/ - -struct LsmFile { - lsm_file *pReal; /* Real underlying file */ - int bLog; /* True for log file. False for db file */ - LsmDb *pDb; /* Database handle that uses this file */ -}; - -static int testEnvFullpath( - lsm_env *pEnv, /* Environment for current LsmDb */ - const char *zFile, /* Relative path name */ - char *zOut, /* Output buffer */ - int *pnOut /* IN/OUT: Size of output buffer */ -){ - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xFullpath(pRealEnv, zFile, zOut, pnOut); -} - -static int testEnvOpen( - lsm_env *pEnv, /* Environment for current LsmDb */ - const char *zFile, /* Name of file to open */ - int flags, - lsm_file **ppFile /* OUT: New file handle object */ -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx; - int rc; /* Return Code */ - LsmFile *pRet; /* The new file handle */ - int nFile; /* Length of string zFile in bytes */ - - nFile = strlen(zFile); - pRet = (LsmFile *)testMalloc(sizeof(LsmFile)); - pRet->pDb = pDb; - pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4)); - - rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal); - if( rc!=LSM_OK ){ - testFree(pRet); - pRet = 0; - } - - *ppFile = (lsm_file *)pRet; - return rc; -} - -static int testEnvRead(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - if( p->pDb->bCrashed ) return LSM_IOERR; - return pRealEnv->xRead(p->pReal, iOff, pData, nData); -} - -static int testEnvWrite(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - LsmDb *pDb = p->pDb; - - if( pDb->bCrashed ) return LSM_IOERR; - - if( pDb->bPrepareCrash ){ - FileData *pData2 = &pDb->aFile[p->bLog]; - int iFirst; - int iLast; - int iSector; - - iFirst = (int)(iOff / pDb->szSector); - iLast = (int)((iOff + nData - 1) / pDb->szSector); - - if( pData2->nSector<(iLast+1) ){ - int nNew = ( ((iLast + 1) + 63) / 64 ) * 64; - assert( nNew>iLast ); - pData2->aSector = (FileSector *)testRealloc( - pData2->aSector, nNew*sizeof(FileSector) - ); - memset(&pData2->aSector[pData2->nSector], - 0, (nNew - pData2->nSector) * sizeof(FileSector) - ); - pData2->nSector = nNew; - } - - for(iSector=iFirst; iSector<=iLast; iSector++){ - if( pData2->aSector[iSector].aOld==0 ){ - u8 *aOld = (u8 *)testMalloc(pDb->szSector); - pRealEnv->xRead( - p->pReal, (lsm_i64)iSector*pDb->szSector, aOld, pDb->szSector - ); - pData2->aSector[iSector].aOld = aOld; - } - } - } - - if( pDb->xWriteHook ){ - int rc; - int nUs; - struct timeval t1; - struct timeval t2; - - gettimeofday(&t1, 0); - assert( nData>0 ); - rc = pRealEnv->xWrite(p->pReal, iOff, pData, nData); - gettimeofday(&t2, 0); - - nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); - pDb->xWriteHook(pDb->pWriteCtx, p->bLog, iOff, nData, nUs); - return rc; - } - - return pRealEnv->xWrite(p->pReal, iOff, pData, nData); -} - -static void doSystemCrash(LsmDb *pDb); - -static int testEnvSync(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - LsmDb *pDb = p->pDb; - FileData *pData = &pDb->aFile[p->bLog]; - int i; - - if( pDb->bCrashed ) return LSM_IOERR; - - if( pDb->nAutoCrash ){ - pDb->nAutoCrash--; - if( pDb->nAutoCrash==0 ){ - doSystemCrash(pDb); - pDb->bCrashed = 1; - return LSM_IOERR; - } - } - - if( pDb->bPrepareCrash ){ - for(i=0; i<pData->nSector; i++){ - testFree(pData->aSector[i].aOld); - pData->aSector[i].aOld = 0; - } - } - - if( pDb->xWriteHook ){ - int rc; - int nUs; - struct timeval t1; - struct timeval t2; - - gettimeofday(&t1, 0); - rc = pRealEnv->xSync(p->pReal); - gettimeofday(&t2, 0); - - nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); - pDb->xWriteHook(pDb->pWriteCtx, p->bLog, 0, 0, nUs); - return rc; - } - - return pRealEnv->xSync(p->pReal); -} - -static int testEnvTruncate(lsm_file *pFile, lsm_i64 iOff){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - if( p->pDb->bCrashed ) return LSM_IOERR; - return pRealEnv->xTruncate(p->pReal, iOff); -} - -static int testEnvSectorSize(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xSectorSize(p->pReal); -} - -static int testEnvRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xRemap(p->pReal, iMin, ppOut, pnOut); -} - -static int testEnvFileid( - lsm_file *pFile, - void *ppOut, - int *pnOut -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xFileid(p->pReal, ppOut, pnOut); -} - -static int testEnvClose(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - - pRealEnv->xClose(p->pReal); - testFree(p); - return LSM_OK; -} - -static int testEnvUnlink(lsm_env *pEnv, const char *zFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - unused_parameter(pEnv); - return pRealEnv->xUnlink(pRealEnv, zFile); -} - -static int testEnvLock(lsm_file *pFile, int iLock, int eType){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - - if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ - return LSM_BUSY; - } - return pRealEnv->xLock(p->pReal, iLock, eType); -} - -static int testEnvTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - - if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ - return LSM_BUSY; - } - return pRealEnv->xTestLock(p->pReal, iLock, nLock, eType); -} - -static int testEnvShmMap(lsm_file *pFile, int iRegion, int sz, void **pp){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xShmMap(p->pReal, iRegion, sz, pp); -} - -static void testEnvShmBarrier(void){ -} - -static int testEnvShmUnmap(lsm_file *pFile, int bDel){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xShmUnmap(p->pReal, bDel); -} - -static int testEnvSleep(lsm_env *pEnv, int us){ - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xSleep(pRealEnv, us); -} - -static void doSystemCrash(LsmDb *pDb){ - lsm_env *pEnv = tdb_lsm_env(); - int iFile; - int iSeed = pDb->aFile[0].nSector + pDb->aFile[1].nSector; - - char *zFile = pDb->zName; - char *zFree = 0; - - for(iFile=0; iFile<2; iFile++){ - lsm_file *pFile = 0; - int i; - - pEnv->xOpen(pEnv, zFile, 0, &pFile); - for(i=0; i<pDb->aFile[iFile].nSector; i++){ - u8 *aOld = pDb->aFile[iFile].aSector[i].aOld; - if( aOld ){ - int iOpt = testPrngValue(iSeed++) % 3; - switch( iOpt ){ - case 0: - break; - - case 1: - testPrngArray(iSeed++, (u32 *)aOld, pDb->szSector/4); - /* Fall-through */ - - case 2: - pEnv->xWrite( - pFile, (lsm_i64)i * pDb->szSector, aOld, pDb->szSector - ); - break; - } - testFree(aOld); - pDb->aFile[iFile].aSector[i].aOld = 0; - } - } - pEnv->xClose(pFile); - zFree = zFile = sqlite3_mprintf("%s-log", pDb->zName); - } - - sqlite3_free(zFree); -} -/* -** End test VFS code. -************************************************************************** -*************************************************************************/ - -/************************************************************************* -************************************************************************** -** Begin test compression hooks. -*/ - -#ifdef HAVE_ZLIB -#include <zlib.h> - -static int testZipBound(void *pCtx, int nSrc){ - return compressBound(nSrc); -} - -static int testZipCompress( - void *pCtx, /* Context pointer */ - char *aOut, int *pnOut, /* OUT: Buffer containing compressed data */ - const char *aIn, int nIn /* Buffer containing input data */ -){ - uLongf n = *pnOut; /* In/out buffer size for compress() */ - int rc; /* compress() return code */ - - rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); - *pnOut = n; - return (rc==Z_OK ? 0 : LSM_ERROR); -} - -static int testZipUncompress( - void *pCtx, /* Context pointer */ - char *aOut, int *pnOut, /* OUT: Buffer containing uncompressed data */ - const char *aIn, int nIn /* Buffer containing input data */ -){ - uLongf n = *pnOut; /* In/out buffer size for uncompress() */ - int rc; /* uncompress() return code */ - - rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); - *pnOut = n; - return (rc==Z_OK ? 0 : LSM_ERROR); -} - -static int testConfigureCompression(lsm_db *pDb){ - static lsm_compress zip = { - 0, /* Context pointer (unused) */ - 1, /* Id value */ - testZipBound, /* xBound method */ - testZipCompress, /* xCompress method */ - testZipUncompress /* xUncompress method */ - }; - return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip); -} -#endif /* ifdef HAVE_ZLIB */ - -/* -** End test compression hooks. -************************************************************************** -*************************************************************************/ - -static int test_lsm_close(TestDb *pTestDb){ - int i; - int rc = LSM_OK; - LsmDb *pDb = (LsmDb *)pTestDb; - - lsm_csr_close(pDb->pCsr); - lsm_close(pDb->db); - - /* If this is a multi-threaded database, wait on the worker threads. */ - mt_shutdown(pDb); - for(i=0; i<pDb->nWorker && rc==LSM_OK; i++){ - rc = pDb->aWorker[i].worker_rc; - } - - for(i=0; i<pDb->aFile[0].nSector; i++){ - testFree(pDb->aFile[0].aSector[i].aOld); - } - testFree(pDb->aFile[0].aSector); - for(i=0; i<pDb->aFile[1].nSector; i++){ - testFree(pDb->aFile[1].aSector[i].aOld); - } - testFree(pDb->aFile[1].aSector); - - memset(pDb, sizeof(LsmDb), 0x11); - testFree((char *)pDb->pBuf); - testFree((char *)pDb); - return rc; -} - -static void mt_signal_worker(LsmDb*, int); - -static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){ - int nSleep = 0; - int nKB; - int rc; - - do { - nKB = 0; - rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB); - if( rc!=LSM_OK || nKB<pDb->nMtMaxCkpt ) break; -#ifdef LSM_MUTEX_PTHREADS - mt_signal_worker(pDb, - (pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ? 0 : 1) - ); -#endif - usleep(5000); - nSleep += 5; - }while( 1 ); - -#if 0 - if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep); -#endif - - return rc; -} - -static int waitOnWorker(LsmDb *pDb){ - int rc; - int nLimit = -1; - int nSleep = 0; - - rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); - do { - int nOld, nNew, rc2; - rc2 = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew); - if( rc2!=LSM_OK ) return rc2; - if( nOld==0 || nNew<(nLimit/2) ) break; -#ifdef LSM_MUTEX_PTHREADS - mt_signal_worker(pDb, 0); -#endif - usleep(5000); - nSleep += 5; - }while( 1 ); - -#if 0 - if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep); -#endif - - return rc; -} - -static int test_lsm_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - LsmDb *pDb = (LsmDb *)pTestDb; - int rc = LSM_OK; - - if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){ - rc = waitOnCheckpointer(pDb, pDb->db); - }else if( - pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK - || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH - ){ - rc = waitOnWorker(pDb); - } - - if( rc==LSM_OK ){ - rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal); - } - return rc; -} - -static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ - LsmDb *pDb = (LsmDb *)pTestDb; - return lsm_delete(pDb->db, pKey, nKey); -} - -static int test_lsm_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - LsmDb *pDb = (LsmDb *)pTestDb; - return lsm_delete_range(pDb->db, pKey1, nKey1, pKey2, nKey2); -} - -static int test_lsm_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - int rc; - LsmDb *pDb = (LsmDb *)pTestDb; - lsm_cursor *csr; - - if( pKey==0 ) return LSM_OK; - - if( pDb->pCsr==0 ){ - rc = lsm_csr_open(pDb->db, &csr); - if( rc!=LSM_OK ) return rc; - }else{ - csr = pDb->pCsr; - } - - rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ); - if( rc==LSM_OK ){ - if( lsm_csr_valid(csr) ){ - const void *pVal; int nVal; - rc = lsm_csr_value(csr, &pVal, &nVal); - if( nVal>pDb->nBuf ){ - testFree(pDb->pBuf); - pDb->pBuf = testMalloc(nVal*2); - pDb->nBuf = nVal*2; - } - memcpy(pDb->pBuf, pVal, nVal); - *ppVal = pDb->pBuf; - *pnVal = nVal; - }else{ - *ppVal = 0; - *pnVal = -1; - } - } - if( pDb->pCsr==0 ){ - lsm_csr_close(csr); - } - return rc; -} - -static int test_lsm_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - LsmDb *pDb = (LsmDb *)pTestDb; - lsm_cursor *csr; - lsm_cursor *csr2 = 0; - int rc; - - if( pDb->pCsr==0 ){ - rc = lsm_csr_open(pDb->db, &csr); - if( rc!=LSM_OK ) return rc; - }else{ - rc = LSM_OK; - csr = pDb->pCsr; - } - - /* To enhance testing, if both pLast and pFirst are defined, seek the - ** cursor to the "end" boundary here. Then the next block seeks it to - ** the "start" ready for the scan. The point is to test that cursors - ** can be reused. */ - if( pLast && pFirst ){ - if( bReverse ){ - rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE); - }else{ - rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE); - } - } - - if( bReverse ){ - if( pLast ){ - rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_LE); - }else{ - rc = lsm_csr_last(csr); - } - }else{ - if( pFirst ){ - rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_GE); - }else{ - rc = lsm_csr_first(csr); - } - } - - while( rc==LSM_OK && lsm_csr_valid(csr) ){ - const void *pKey; int nKey; - const void *pVal; int nVal; - int cmp; - - lsm_csr_key(csr, &pKey, &nKey); - lsm_csr_value(csr, &pVal, &nVal); - - if( bReverse && pFirst ){ - cmp = memcmp(pFirst, pKey, MIN(nKey, nFirst)); - if( cmp>0 || (cmp==0 && nFirst>nKey) ) break; - }else if( bReverse==0 && pLast ){ - cmp = memcmp(pLast, pKey, MIN(nKey, nLast)); - if( cmp<0 || (cmp==0 && nLast<nKey) ) break; - } - - xCallback(pCtx, (void *)pKey, nKey, (void *)pVal, nVal); - - if( bReverse ){ - rc = lsm_csr_prev(csr); - }else{ - rc = lsm_csr_next(csr); - } - } - - if( pDb->pCsr==0 ){ - lsm_csr_close(csr); - } - return rc; -} - -static int test_lsm_begin(TestDb *pTestDb, int iLevel){ - int rc = LSM_OK; - LsmDb *pDb = (LsmDb *)pTestDb; - - /* iLevel==0 is a no-op. */ - if( iLevel==0 ) return 0; - - if( pDb->pCsr==0 ) rc = lsm_csr_open(pDb->db, &pDb->pCsr); - if( rc==LSM_OK && iLevel>1 ){ - rc = lsm_begin(pDb->db, iLevel-1); - } - - return rc; -} -static int test_lsm_commit(TestDb *pTestDb, int iLevel){ - LsmDb *pDb = (LsmDb *)pTestDb; - - /* If iLevel==0, close any open read transaction */ - if( iLevel==0 && pDb->pCsr ){ - lsm_csr_close(pDb->pCsr); - pDb->pCsr = 0; - } - - /* If iLevel==0, close any open read transaction */ - return lsm_commit(pDb->db, MAX(0, iLevel-1)); -} -static int test_lsm_rollback(TestDb *pTestDb, int iLevel){ - LsmDb *pDb = (LsmDb *)pTestDb; - - /* If iLevel==0, close any open read transaction */ - if( iLevel==0 && pDb->pCsr ){ - lsm_csr_close(pDb->pCsr); - pDb->pCsr = 0; - } - - return lsm_rollback(pDb->db, MAX(0, iLevel-1)); -} - -/* -** A log message callback registered with lsm connections. Prints all -** messages to stderr. -*/ -static void xLog(void *pCtx, int rc, const char *z){ - unused_parameter(rc); - /* fprintf(stderr, "lsm: rc=%d \"%s\"\n", rc, z); */ - if( pCtx ) fprintf(stderr, "%s: ", (char *)pCtx); - fprintf(stderr, "%s\n", z); - fflush(stderr); -} - -static void xWorkHook(lsm_db *db, void *pArg){ - LsmDb *p = (LsmDb *)pArg; - if( p->xWork ) p->xWork(db, p->pWorkCtx); -} - -#define TEST_NO_RECOVERY -1 -#define TEST_COMPRESSION -3 - -#define TEST_MT_MODE -2 -#define TEST_MT_MIN_CKPT -4 -#define TEST_MT_MAX_CKPT -5 - - -int test_lsm_config_str( - LsmDb *pLsm, - lsm_db *db, - int bWorker, - const char *zStr, - int *pnThread -){ - struct CfgParam { - const char *zParam; - int bWorker; - int eParam; - } aParam[] = { - { "autoflush", 0, LSM_CONFIG_AUTOFLUSH }, - { "page_size", 0, LSM_CONFIG_PAGE_SIZE }, - { "block_size", 0, LSM_CONFIG_BLOCK_SIZE }, - { "safety", 0, LSM_CONFIG_SAFETY }, - { "autowork", 0, LSM_CONFIG_AUTOWORK }, - { "autocheckpoint", 0, LSM_CONFIG_AUTOCHECKPOINT }, - { "mmap", 0, LSM_CONFIG_MMAP }, - { "use_log", 0, LSM_CONFIG_USE_LOG }, - { "automerge", 0, LSM_CONFIG_AUTOMERGE }, - { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, - { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, - { "worker_automerge", 1, LSM_CONFIG_AUTOMERGE }, - { "test_no_recovery", 0, TEST_NO_RECOVERY }, - { "bg_min_ckpt", 0, TEST_NO_RECOVERY }, - - { "mt_mode", 0, TEST_MT_MODE }, - { "mt_min_ckpt", 0, TEST_MT_MIN_CKPT }, - { "mt_max_ckpt", 0, TEST_MT_MAX_CKPT }, - -#ifdef HAVE_ZLIB - { "compression", 0, TEST_COMPRESSION }, -#endif - { 0, 0 } - }; - const char *z = zStr; - int nThread = 1; - - if( zStr==0 ) return 0; - - assert( db ); - while( z[0] ){ - const char *zStart; - - /* Skip whitespace */ - while( *z==' ' ) z++; - zStart = z; - - while( *z && *z!='=' ) z++; - if( *z ){ - int eParam; - int i; - int iVal; - int iMul = 1; - int rc; - char zParam[32]; - int nParam = z-zStart; - if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; - - memcpy(zParam, zStart, nParam); - zParam[nParam] = '\0'; - rc = testArgSelect(aParam, "param", zParam, &i); - if( rc!=0 ) return rc; - eParam = aParam[i].eParam; - - z++; - zStart = z; - while( *z>='0' && *z<='9' ) z++; - if( *z=='k' || *z=='K' ){ - iMul = 1; - z++; - }else if( *z=='M' || *z=='M' ){ - iMul = 1024; - z++; - } - nParam = z-zStart; - if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; - memcpy(zParam, zStart, nParam); - zParam[nParam] = '\0'; - iVal = atoi(zParam) * iMul; - - if( eParam>0 ){ - if( bWorker || aParam[i].bWorker==0 ){ - lsm_config(db, eParam, &iVal); - } - }else{ - switch( eParam ){ - case TEST_NO_RECOVERY: - if( pLsm ) pLsm->bNoRecovery = iVal; - break; - case TEST_MT_MODE: - if( pLsm ) nThread = iVal; - break; - case TEST_MT_MIN_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal*1024; - break; - case TEST_MT_MAX_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal*1024; - break; -#ifdef HAVE_ZLIB - case TEST_COMPRESSION: - testConfigureCompression(db); - break; -#endif - } - } - }else if( z!=zStart ){ - goto syntax_error; - } - } - - if( pnThread ) *pnThread = nThread; - if( pLsm && pLsm->nMtMaxCkpt < pLsm->nMtMinCkpt ){ - pLsm->nMtMinCkpt = pLsm->nMtMaxCkpt; - } - - return 0; - syntax_error: - testPrintError("syntax error at: \"%s\"\n", z); - return 1; -} - -int tdb_lsm_config_str(TestDb *pDb, const char *zStr){ - int rc = 0; - if( tdb_lsm(pDb) ){ -#ifdef LSM_MUTEX_PTHREADS - int i; -#endif - LsmDb *pLsm = (LsmDb *)pDb; - - rc = test_lsm_config_str(pLsm, pLsm->db, 0, zStr, 0); -#ifdef LSM_MUTEX_PTHREADS - for(i=0; rc==0 && i<pLsm->nWorker; i++){ - rc = test_lsm_config_str(0, pLsm->aWorker[i].pWorker, 1, zStr, 0); - } -#endif - } - return rc; -} - -int tdb_lsm_configure(lsm_db *db, const char *zConfig){ - return test_lsm_config_str(0, db, 0, zConfig, 0); -} - -static int testLsmStartWorkers(LsmDb *, int, const char *, const char *); - -static int testLsmOpen( - const char *zCfg, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods LsmMethods = { - test_lsm_close, - test_lsm_write, - test_lsm_delete, - test_lsm_delete_range, - test_lsm_fetch, - test_lsm_scan, - test_lsm_begin, - test_lsm_commit, - test_lsm_rollback - }; - - int rc; - int nFilename; - LsmDb *pDb; - - /* If the bClear flag is set, delete any existing database. */ - assert( zFilename); - if( bClear ) testDeleteLsmdb(zFilename); - nFilename = strlen(zFilename); - - pDb = (LsmDb *)testMalloc(sizeof(LsmDb) + nFilename + 1); - memset(pDb, 0, sizeof(LsmDb)); - pDb->base.pMethods = &LsmMethods; - pDb->zName = (char *)&pDb[1]; - memcpy(pDb->zName, zFilename, nFilename + 1); - - /* Default the sector size used for crash simulation to 512 bytes. - ** Todo: There should be an OS method to obtain this value - just as - ** there is in SQLite. For now, LSM assumes that it is smaller than - ** the page size (default 4KB). - */ - pDb->szSector = 256; - - /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */ - pDb->nMtMinCkpt = LSMTEST_DFLT_MT_MIN_CKPT; - pDb->nMtMaxCkpt = LSMTEST_DFLT_MT_MAX_CKPT; - - memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env)); - pDb->env.pVfsCtx = (void *)pDb; - pDb->env.xFullpath = testEnvFullpath; - pDb->env.xOpen = testEnvOpen; - pDb->env.xRead = testEnvRead; - pDb->env.xWrite = testEnvWrite; - pDb->env.xTruncate = testEnvTruncate; - pDb->env.xSync = testEnvSync; - pDb->env.xSectorSize = testEnvSectorSize; - pDb->env.xRemap = testEnvRemap; - pDb->env.xFileid = testEnvFileid; - pDb->env.xClose = testEnvClose; - pDb->env.xUnlink = testEnvUnlink; - pDb->env.xLock = testEnvLock; - pDb->env.xTestLock = testEnvTestLock; - pDb->env.xShmBarrier = testEnvShmBarrier; - pDb->env.xShmMap = testEnvShmMap; - pDb->env.xShmUnmap = testEnvShmUnmap; - pDb->env.xSleep = testEnvSleep; - - rc = lsm_new(&pDb->env, &pDb->db); - if( rc==LSM_OK ){ - int nThread = 1; - lsm_config_log(pDb->db, xLog, 0); - lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb); - - rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread); - if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename); - - pDb->eMode = nThread; -#ifdef LSM_MUTEX_PTHREADS - if( rc==LSM_OK && nThread>1 ){ - testLsmStartWorkers(pDb, nThread, zFilename, zCfg); - } -#endif - - if( rc!=LSM_OK ){ - test_lsm_close((TestDb *)pDb); - pDb = 0; - } - } - - *ppDb = (TestDb *)pDb; - return rc; -} - -int test_lsm_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return testLsmOpen(zSpec, zFilename, bClear, ppDb); -} - -int test_lsm_small_open( - const char *zSpec, - const char *zFile, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "page_size=256 block_size=64 mmap=1024"; - return testLsmOpen(zCfg, zFile, bClear, ppDb); -} - -int test_lsm_lomem_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - /* "max_freelist=4 autocheckpoint=32" */ - const char *zCfg = - "page_size=256 block_size=64 autoflush=16 " - "autocheckpoint=32" - "mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_lomem2_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - /* "max_freelist=4 autocheckpoint=32" */ - const char *zCfg = - "page_size=512 block_size=64 autoflush=0 mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_zip_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = - "page_size=256 block_size=64 autoflush=16 " - "autocheckpoint=32 compression=1 mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -lsm_db *tdb_lsm(TestDb *pDb){ - if( pDb->pMethods->xClose==test_lsm_close ){ - return ((LsmDb *)pDb)->db; - } - return 0; -} - -int tdb_lsm_multithread(TestDb *pDb){ - int ret = 0; - if( tdb_lsm(pDb) ){ - ret = ((LsmDb*)pDb)->eMode!=LSMTEST_MODE_SINGLETHREAD; - } - return ret; -} - -void tdb_lsm_enable_log(TestDb *pDb, int bEnable){ - lsm_db *db = tdb_lsm(pDb); - if( db ){ - lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client"); - } -} - -void tdb_lsm_application_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bCrashed = 1; - } -} - -void tdb_lsm_prepare_system_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bPrepareCrash = 1; - } -} - -void tdb_lsm_system_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bCrashed = 1; - doSystemCrash(p); - } -} - -void tdb_lsm_safety(TestDb *pDb, int eMode){ - assert( eMode==LSM_SAFETY_OFF - || eMode==LSM_SAFETY_NORMAL - || eMode==LSM_SAFETY_FULL - ); - if( tdb_lsm(pDb) ){ - int iParam = eMode; - LsmDb *p = (LsmDb *)pDb; - lsm_config(p->db, LSM_CONFIG_SAFETY, &iParam); - } -} - -void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync){ - assert( iSync>0 ); - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->nAutoCrash = iSync; - p->bPrepareCrash = 1; - } -} - -void tdb_lsm_config_work_hook( - TestDb *pDb, - void (*xWork)(lsm_db *, void *), - void *pWorkCtx -){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->xWork = xWork; - p->pWorkCtx = pWorkCtx; - } -} - -void tdb_lsm_write_hook( - TestDb *pDb, - void (*xWrite)(void *, int, lsm_i64, int, int), - void *pWriteCtx -){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->xWriteHook = xWrite; - p->pWriteCtx = pWriteCtx; - } -} - -int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb){ - return testLsmOpen(zCfg, zDb, bClear, ppDb); -} - -#ifdef LSM_MUTEX_PTHREADS - -/* -** Signal worker thread iWorker that there may be work to do. -*/ -static void mt_signal_worker(LsmDb *pDb, int iWorker){ - LsmWorker *p = &pDb->aWorker[iWorker]; - pthread_mutex_lock(&p->worker_mutex); - p->bDoWork = 1; - pthread_cond_signal(&p->worker_cond); - pthread_mutex_unlock(&p->worker_mutex); -} - -/* -** This routine is used as the main() for all worker threads. -*/ -static void *worker_main(void *pArg){ - LsmWorker *p = (LsmWorker *)pArg; - lsm_db *pWorker; /* Connection to access db through */ - - pthread_mutex_lock(&p->worker_mutex); - while( (pWorker = p->pWorker) ){ - int rc = LSM_OK; - - /* Do some work. If an error occurs, exit. */ - - pthread_mutex_unlock(&p->worker_mutex); - if( p->eType==LSMTEST_THREAD_CKPT ){ - int nKB = 0; - rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nKB); - if( rc==LSM_OK && nKB>=p->pDb->nMtMinCkpt ){ - rc = lsm_checkpoint(pWorker, 0); - } - }else{ - int nWrite; - do { - - if( p->eType==LSMTEST_THREAD_WORKER ){ - waitOnCheckpointer(p->pDb, pWorker); - } - - nWrite = 0; - rc = lsm_work(pWorker, 0, 256, &nWrite); - - if( p->eType==LSMTEST_THREAD_WORKER && nWrite ){ - mt_signal_worker(p->pDb, 1); - } - }while( nWrite && p->pWorker ); - } - pthread_mutex_lock(&p->worker_mutex); - - if( rc!=LSM_OK && rc!=LSM_BUSY ){ - p->worker_rc = rc; - break; - } - - /* The thread will wake up when it is signaled either because another - ** thread has created some work for this one or because the connection - ** is being closed. */ - if( p->pWorker && p->bDoWork==0 ){ - pthread_cond_wait(&p->worker_cond, &p->worker_mutex); - } - p->bDoWork = 0; - } - pthread_mutex_unlock(&p->worker_mutex); - - return 0; -} - - -static void mt_stop_worker(LsmDb *pDb, int iWorker){ - LsmWorker *p = &pDb->aWorker[iWorker]; - if( p->pWorker ){ - void *pDummy; - lsm_db *pWorker; - - /* Signal the worker to stop */ - pthread_mutex_lock(&p->worker_mutex); - pWorker = p->pWorker; - p->pWorker = 0; - pthread_cond_signal(&p->worker_cond); - pthread_mutex_unlock(&p->worker_mutex); - - /* Join the worker thread. */ - pthread_join(p->worker_thread, &pDummy); - - /* Free resources allocated in mt_start_worker() */ - pthread_cond_destroy(&p->worker_cond); - pthread_mutex_destroy(&p->worker_mutex); - lsm_close(pWorker); - } -} - -static void mt_shutdown(LsmDb *pDb){ - int i; - for(i=0; i<pDb->nWorker; i++){ - mt_stop_worker(pDb, i); - } -} - -/* -** This callback is invoked by LSM when the client database writes to -** the database file (i.e. to flush the contents of the in-memory tree). -** This implies there may be work to do on the database, so signal -** the worker threads. -*/ -static void mt_client_work_hook(lsm_db *db, void *pArg){ - LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ - - /* Invoke the user level work-hook, if any. */ - if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); - - /* Wake up worker thread 0. */ - mt_signal_worker(pDb, 0); -} - -static void mt_worker_work_hook(lsm_db *db, void *pArg){ - LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ - - /* Invoke the user level work-hook, if any. */ - if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); -} - -/* -** Launch worker thread iWorker for database connection pDb. -*/ -static int mt_start_worker( - LsmDb *pDb, /* Main database structure */ - int iWorker, /* Worker number to start */ - const char *zFilename, /* File name of database to open */ - const char *zCfg, /* Connection configuration string */ - int eType /* Type of worker thread */ -){ - int rc = 0; /* Return code */ - LsmWorker *p; /* Object to initialize */ - - assert( iWorker<pDb->nWorker ); - assert( eType==LSMTEST_THREAD_CKPT - || eType==LSMTEST_THREAD_WORKER - || eType==LSMTEST_THREAD_WORKER_AC - ); - - p = &pDb->aWorker[iWorker]; - p->eType = eType; - p->pDb = pDb; - - /* Open the worker connection */ - if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker); - if( zCfg ){ - test_lsm_config_str(pDb, p->pWorker, 1, zCfg, 0); - } - if( rc==0 ) rc = lsm_open(p->pWorker, zFilename); - lsm_config_log(p->pWorker, xLog, (void *)"worker"); - - /* Configure the work-hook */ - if( rc==0 ){ - lsm_config_work_hook(p->pWorker, mt_worker_work_hook, (void *)pDb); - } - - if( eType==LSMTEST_THREAD_WORKER ){ - test_lsm_config_str(0, p->pWorker, 1, "autocheckpoint=0", 0); - } - - /* Kick off the worker thread. */ - if( rc==0 ) rc = pthread_cond_init(&p->worker_cond, 0); - if( rc==0 ) rc = pthread_mutex_init(&p->worker_mutex, 0); - if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p); - - return rc; -} - - -static int testLsmStartWorkers( - LsmDb *pDb, int eModel, const char *zFilename, const char *zCfg -){ - int rc; - - if( eModel<1 || eModel>4 ) return 1; - if( eModel==1 ) return 0; - - /* Configure a work-hook for the client connection. Worker 0 is signalled - ** every time the users connection writes to the database. */ - lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb); - - /* Allocate space for two worker connections. They may not both be - ** used, but both are allocated. */ - pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * 2); - memset(pDb->aWorker, 0, sizeof(LsmWorker) * 2); - - switch( eModel ){ - case LSMTEST_MODE_BACKGROUND_CKPT: - pDb->nWorker = 1; - test_lsm_config_str(0, pDb->db, 0, "autocheckpoint=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_CKPT); - break; - - case LSMTEST_MODE_BACKGROUND_WORK: - pDb->nWorker = 1; - test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER_AC); - break; - - case LSMTEST_MODE_BACKGROUND_BOTH: - pDb->nWorker = 2; - test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER); - if( rc==0 ){ - rc = mt_start_worker(pDb, 1, zFilename, zCfg, LSMTEST_THREAD_CKPT); - } - break; - } - - return rc; -} - - -int test_lsm_mt2( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "mt_mode=2"; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_mt3( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "mt_mode=4"; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -#else -static void mt_shutdown(LsmDb *pDb) { - unused_parameter(pDb); -} -int test_lsm_mt(const char *zFilename, int bClear, TestDb **ppDb){ - unused_parameter(zFilename); - unused_parameter(bClear); - unused_parameter(ppDb); - testPrintError("threads unavailable - recompile with LSM_MUTEX_PTHREADS\n"); - return 1; -} -#endif diff --git a/ext/lsm1/lsm-test/lsmtest_tdb4.c b/ext/lsm1/lsm-test/lsmtest_tdb4.c deleted file mode 100644 index 1f9292852..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb4.c +++ /dev/null @@ -1,980 +0,0 @@ - -/* -** This file contains the TestDb bt wrapper. -*/ - -#include "lsmtest_tdb.h" -#include "lsmtest.h" -#include <unistd.h> -#include "bt.h" - -#include <pthread.h> - -typedef struct BtDb BtDb; -typedef struct BtFile BtFile; - -/* Background checkpointer interface (see implementations below). */ -typedef struct bt_ckpter bt_ckpter; -static int bgc_attach(BtDb *pDb, const char*); -static int bgc_detach(BtDb *pDb); - -/* -** Each database or log file opened by a database handle is wrapped by -** an object of the following type. -*/ -struct BtFile { - BtDb *pBt; /* Database handle that opened this file */ - bt_env *pVfs; /* Underlying VFS */ - bt_file *pFile; /* File handle belonging to underlying VFS */ - int nSectorSize; /* Size of sectors in bytes */ - int nSector; /* Allocated size of nSector array */ - u8 **apSector; /* Original sector data */ -}; - -/* -** nCrashSync: -** If this value is non-zero, then a "crash-test" is running. If -** nCrashSync==1, then the crash is simulated during the very next -** call to the xSync() VFS method (on either the db or log file). -** If nCrashSync==2, the following call to xSync(), and so on. -** -** bCrash: -** After a crash is simulated, this variable is set. Any subsequent -** attempts to write to a file or modify the file system in any way -** fail once this is set. All the caller can do is close the connection. -** -** bFastInsert: -** If this variable is set to true, then a BT_CONTROL_FAST_INSERT_OP -** control is issued before each callto BtReplace() or BtCsrOpen(). -*/ -struct BtDb { - TestDb base; /* Base class */ - bt_db *pBt; /* bt database handle */ - sqlite4_env *pEnv; /* SQLite environment (for malloc/free) */ - bt_env *pVfs; /* Underlying VFS */ - int bFastInsert; /* True to use fast-insert */ - - /* Space for bt_fetch() results */ - u8 *aBuffer; /* Space to store results */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ - int nRef; - - /* Background checkpointer used by mt connections */ - bt_ckpter *pCkpter; - - /* Stuff used for crash test simulation */ - BtFile *apFile[2]; /* Database and log files used by pBt */ - bt_env env; /* Private VFS for this object */ - int nCrashSync; /* Number of syncs until crash (see above) */ - int bCrash; /* True once a crash has been simulated */ -}; - -static int btVfsFullpath( - sqlite4_env *pEnv, - bt_env *pVfs, - const char *z, - char **pzOut -){ - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - if( pBt->bCrash ) return SQLITE4_IOERR; - return pBt->pVfs->xFullpath(pEnv, pBt->pVfs, z, pzOut); -} - -static int btVfsOpen( - sqlite4_env *pEnv, - bt_env *pVfs, - const char *zFile, - int flags, bt_file **ppFile -){ - BtFile *p; - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - int rc; - - if( pBt->bCrash ) return SQLITE4_IOERR; - - p = (BtFile*)testMalloc(sizeof(BtFile)); - if( !p ) return SQLITE4_NOMEM; - if( flags & BT_OPEN_DATABASE ){ - pBt->apFile[0] = p; - }else if( flags & BT_OPEN_LOG ){ - pBt->apFile[1] = p; - } - if( (flags & BT_OPEN_SHARED)==0 ){ - p->pBt = pBt; - } - p->pVfs = pBt->pVfs; - - rc = pBt->pVfs->xOpen(pEnv, pVfs, zFile, flags, &p->pFile); - if( rc!=SQLITE4_OK ){ - testFree(p); - p = 0; - }else{ - pBt->nRef++; - } - - *ppFile = (bt_file*)p; - return rc; -} - -static int btVfsSize(bt_file *pFile, sqlite4_int64 *piRes){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xSize(p->pFile, piRes); -} - -static int btVfsRead(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xRead(p->pFile, iOff, pBuf, nBuf); -} - -static int btFlushSectors(BtFile *p, int iFile){ - sqlite4_int64 iSz; - int rc; - int i; - u8 *aTmp = 0; - - rc = p->pBt->pVfs->xSize(p->pFile, &iSz); - for(i=0; rc==SQLITE4_OK && i<p->nSector; i++){ - if( p->pBt->bCrash && p->apSector[i] ){ - - /* The system is simulating a crash. There are three choices for - ** this sector: - ** - ** 1) Leave it as it is (simulating a successful write), - ** 2) Restore the original data (simulating a lost write), - ** 3) Populate the disk sector with garbage data. - */ - sqlite4_int64 iSOff = p->nSectorSize*i; - int nWrite = MIN(p->nSectorSize, iSz - iSOff); - - if( nWrite ){ - u8 *aWrite = 0; - int iOpt = (testPrngValue(i) % 3) + 1; - if( iOpt==1 ){ - aWrite = p->apSector[i]; - }else if( iOpt==3 ){ - if( aTmp==0 ) aTmp = testMalloc(p->nSectorSize); - aWrite = aTmp; - testPrngArray(i*13, (u32*)aWrite, nWrite/sizeof(u32)); - } - -#if 0 -fprintf(stderr, "handle sector %d of %s with %s\n", i, - iFile==0 ? "db" : "log", - iOpt==1 ? "rollback" : iOpt==2 ? "write" : "omit" -); -fflush(stderr); -#endif - - if( aWrite ){ - rc = p->pBt->pVfs->xWrite(p->pFile, iSOff, aWrite, nWrite); - } - } - } - testFree(p->apSector[i]); - p->apSector[i] = 0; - } - - testFree(aTmp); - return rc; -} - -static int btSaveSectors(BtFile *p, sqlite4_int64 iOff, int nBuf){ - int rc; - sqlite4_int64 iSz; /* Size of file on disk */ - int iFirst; /* First sector affected */ - int iSector; /* Current sector */ - int iLast; /* Last sector affected */ - - if( p->nSectorSize==0 ){ - p->nSectorSize = p->pBt->pVfs->xSectorSize(p->pFile); - if( p->nSectorSize<512 ) p->nSectorSize = 512; - } - iLast = (iOff+nBuf-1) / p->nSectorSize; - iFirst = iOff / p->nSectorSize; - - rc = p->pBt->pVfs->xSize(p->pFile, &iSz); - for(iSector=iFirst; rc==SQLITE4_OK && iSector<=iLast; iSector++){ - int nRead; - sqlite4_int64 iSOff = iSector * p->nSectorSize; - u8 *aBuf = testMalloc(p->nSectorSize); - nRead = MIN(p->nSectorSize, (iSz - iSOff)); - if( nRead>0 ){ - rc = p->pBt->pVfs->xRead(p->pFile, iSOff, aBuf, nRead); - } - - while( rc==SQLITE4_OK && iSector>=p->nSector ){ - int nNew = p->nSector + 32; - u8 **apNew = (u8**)testMalloc(nNew * sizeof(u8*)); - memcpy(apNew, p->apSector, p->nSector*sizeof(u8*)); - testFree(p->apSector); - p->apSector = apNew; - p->nSector = nNew; - } - - p->apSector[iSector] = aBuf; - } - - return rc; -} - -static int btVfsWrite(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - if( p->pBt && p->pBt->nCrashSync ){ - btSaveSectors(p, iOff, nBuf); - } - return p->pVfs->xWrite(p->pFile, iOff, pBuf, nBuf); -} - -static int btVfsTruncate(bt_file *pFile, sqlite4_int64 iOff){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xTruncate(p->pFile, iOff); -} - -static int btVfsSync(bt_file *pFile){ - int rc = SQLITE4_OK; - BtFile *p = (BtFile*)pFile; - BtDb *pBt = p->pBt; - - if( pBt ){ - if( pBt->bCrash ) return SQLITE4_IOERR; - if( pBt->nCrashSync ){ - pBt->nCrashSync--; - pBt->bCrash = (pBt->nCrashSync==0); - if( pBt->bCrash ){ - btFlushSectors(pBt->apFile[0], 0); - btFlushSectors(pBt->apFile[1], 1); - rc = SQLITE4_IOERR; - }else{ - btFlushSectors(p, 0); - } - } - } - - if( rc==SQLITE4_OK ){ - rc = p->pVfs->xSync(p->pFile); - } - return rc; -} - -static int btVfsSectorSize(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - return p->pVfs->xSectorSize(p->pFile); -} - -static void btDeref(BtDb *p){ - p->nRef--; - assert( p->nRef>=0 ); - if( p->nRef<=0 ) testFree(p); -} - -static int btVfsClose(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - BtDb *pBt = p->pBt; - int rc; - if( pBt ){ - btFlushSectors(p, 0); - if( p==pBt->apFile[0] ) pBt->apFile[0] = 0; - if( p==pBt->apFile[1] ) pBt->apFile[1] = 0; - } - testFree(p->apSector); - rc = p->pVfs->xClose(p->pFile); -#if 0 - btDeref(p->pBt); -#endif - testFree(p); - return rc; -} - -static int btVfsUnlink(sqlite4_env *pEnv, bt_env *pVfs, const char *zFile){ - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - if( pBt->bCrash ) return SQLITE4_IOERR; - return pBt->pVfs->xUnlink(pEnv, pBt->pVfs, zFile); -} - -static int btVfsLock(bt_file *pFile, int iLock, int eType){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xLock(p->pFile, iLock, eType); -} - -static int btVfsTestLock(bt_file *pFile, int iLock, int nLock, int eType){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xTestLock(p->pFile, iLock, nLock, eType); -} - -static int btVfsShmMap(bt_file *pFile, int iChunk, int sz, void **ppOut){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xShmMap(p->pFile, iChunk, sz, ppOut); -} - -static void btVfsShmBarrier(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - return p->pVfs->xShmBarrier(p->pFile); -} - -static int btVfsShmUnmap(bt_file *pFile, int bDelete){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xShmUnmap(p->pFile, bDelete); -} - -static int bt_close(TestDb *pTestDb){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtClose(p->pBt); - free(p->aBuffer); - if( p->apFile[0] ) p->apFile[0]->pBt = 0; - if( p->apFile[1] ) p->apFile[1]->pBt = 0; - bgc_detach(p); - testFree(p); - return rc; -} - -static int btMinTransaction(BtDb *p, int iMin, int *piLevel){ - int iLevel; - int rc = SQLITE4_OK; - - iLevel = sqlite4BtTransactionLevel(p->pBt); - if( iLevel<iMin ){ - rc = sqlite4BtBegin(p->pBt, iMin); - *piLevel = iLevel; - }else{ - *piLevel = -1; - } - - return rc; -} -static int btRestoreTransaction(BtDb *p, int iLevel, int rcin){ - int rc = rcin; - if( iLevel>=0 ){ - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCommit(p->pBt, iLevel); - }else{ - sqlite4BtRollback(p->pBt, iLevel); - } - assert( iLevel==sqlite4BtTransactionLevel(p->pBt) ); - } - return rc; -} - -static int bt_write(TestDb *pTestDb, void *pK, int nK, void *pV, int nV){ - BtDb *p = (BtDb*)pTestDb; - int iLevel; - int rc; - - rc = btMinTransaction(p, 2, &iLevel); - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtReplace(p->pBt, pK, nK, pV, nV); - rc = btRestoreTransaction(p, iLevel, rc); - } - return rc; -} - -static int bt_delete(TestDb *pTestDb, void *pK, int nK){ - return bt_write(pTestDb, pK, nK, 0, -1); -} - -static int bt_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int rc = SQLITE4_OK; - int iLevel; - - rc = btMinTransaction(p, 2, &iLevel); - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - } - while( rc==SQLITE4_OK ){ - const void *pK; - int n; - int nCmp; - int res; - - rc = sqlite4BtCsrSeek(pCsr, pKey1, nKey1, BT_SEEK_GE); - if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; - if( rc!=SQLITE4_OK ) break; - - rc = sqlite4BtCsrKey(pCsr, &pK, &n); - if( rc!=SQLITE4_OK ) break; - - nCmp = MIN(n, nKey1); - res = memcmp(pKey1, pK, nCmp); - assert( res<0 || (res==0 && nKey1<=n) ); - if( res==0 && nKey1==n ){ - rc = sqlite4BtCsrNext(pCsr); - if( rc!=SQLITE4_OK ) break; - rc = sqlite4BtCsrKey(pCsr, &pK, &n); - if( rc!=SQLITE4_OK ) break; - } - - nCmp = MIN(n, nKey2); - res = memcmp(pKey2, pK, nCmp); - if( res<0 || (res==0 && nKey2<=n) ) break; - - rc = sqlite4BtDelete(pCsr); - } - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - - sqlite4BtCsrClose(pCsr); - - rc = btRestoreTransaction(p, iLevel, rc); - return rc; -} - -static int bt_fetch( - TestDb *pTestDb, - void *pK, int nK, - void **ppVal, int *pnVal -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int iLevel; - int rc = SQLITE4_OK; - - iLevel = sqlite4BtTransactionLevel(p->pBt); - if( iLevel==0 ){ - rc = sqlite4BtBegin(p->pBt, 1); - if( rc!=SQLITE4_OK ) return rc; - } - - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCsrSeek(pCsr, pK, nK, BT_SEEK_EQ); - if( rc==SQLITE4_OK ){ - const void *pV = 0; - int nV = 0; - rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV); - if( rc==SQLITE4_OK ){ - if( nV>p->nBuffer ){ - free(p->aBuffer); - p->aBuffer = (u8*)malloc(nV*2); - p->nBuffer = nV*2; - } - memcpy(p->aBuffer, pV, nV); - *pnVal = nV; - *ppVal = (void*)(p->aBuffer); - } - - }else if( rc==SQLITE4_INEXACT || rc==SQLITE4_NOTFOUND ){ - *ppVal = 0; - *pnVal = -1; - rc = SQLITE4_OK; - } - sqlite4BtCsrClose(pCsr); - } - - if( iLevel==0 ) sqlite4BtCommit(p->pBt, 0); - return rc; -} - -static int bt_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int rc; - int iLevel; - - rc = btMinTransaction(p, 1, &iLevel); - - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - } - if( rc==SQLITE4_OK ){ - if( bReverse ){ - if( pLast ){ - rc = sqlite4BtCsrSeek(pCsr, pLast, nLast, BT_SEEK_LE); - }else{ - rc = sqlite4BtCsrLast(pCsr); - } - }else{ - rc = sqlite4BtCsrSeek(pCsr, pFirst, nFirst, BT_SEEK_GE); - } - if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; - - while( rc==SQLITE4_OK ){ - const void *pK = 0; int nK = 0; - const void *pV = 0; int nV = 0; - - rc = sqlite4BtCsrKey(pCsr, &pK, &nK); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV); - } - - if( rc!=SQLITE4_OK ) break; - if( bReverse ){ - if( pFirst ){ - int res; - int nCmp = MIN(nK, nFirst); - res = memcmp(pFirst, pK, nCmp); - if( res>0 || (res==0 && nK<nFirst) ) break; - } - }else{ - if( pLast ){ - int res; - int nCmp = MIN(nK, nLast); - res = memcmp(pLast, pK, nCmp); - if( res<0 || (res==0 && nK>nLast) ) break; - } - } - - xCallback(pCtx, (void*)pK, nK, (void*)pV, nV); - if( bReverse ){ - rc = sqlite4BtCsrPrev(pCsr); - }else{ - rc = sqlite4BtCsrNext(pCsr); - } - } - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - - sqlite4BtCsrClose(pCsr); - } - - rc = btRestoreTransaction(p, iLevel, rc); - return rc; -} - -static int bt_begin(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtBegin(p->pBt, iLvl); - return rc; -} - -static int bt_commit(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtCommit(p->pBt, iLvl); - return rc; -} - -static int bt_rollback(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtRollback(p->pBt, iLvl); - return rc; -} - -static int testParseOption( - const char **pzIn, /* IN/OUT: pointer to next option */ - const char **pzOpt, /* OUT: nul-terminated option name */ - const char **pzArg, /* OUT: nul-terminated option argument */ - char *pSpace /* Temporary space for output params */ -){ - const char *p = *pzIn; - const char *pStart; - int n; - - char *pOut = pSpace; - - while( *p==' ' ) p++; - pStart = p; - while( *p && *p!='=' ) p++; - if( *p==0 ) return 1; - - n = (p - pStart); - memcpy(pOut, pStart, n); - *pzOpt = pOut; - pOut += n; - *pOut++ = '\0'; - - p++; - pStart = p; - while( *p && *p!=' ' ) p++; - n = (p - pStart); - - memcpy(pOut, pStart, n); - *pzArg = pOut; - pOut += n; - *pOut++ = '\0'; - - *pzIn = p; - return 0; -} - -static int testParseInt(const char *z, int *piVal){ - int i = 0; - const char *p = z; - - while( *p>='0' && *p<='9' ){ - i = i*10 + (*p - '0'); - p++; - } - if( *p=='K' || *p=='k' ){ - i = i * 1024; - p++; - }else if( *p=='M' || *p=='m' ){ - i = i * 1024 * 1024; - p++; - } - - if( *p ) return SQLITE4_ERROR; - *piVal = i; - return SQLITE4_OK; -} - -static int testBtConfigure(BtDb *pDb, const char *zCfg, int *pbMt){ - int rc = SQLITE4_OK; - - if( zCfg ){ - struct CfgParam { - const char *zParam; - int eParam; - } aParam[] = { - { "safety", BT_CONTROL_SAFETY }, - { "autockpt", BT_CONTROL_AUTOCKPT }, - { "multiproc", BT_CONTROL_MULTIPROC }, - { "blksz", BT_CONTROL_BLKSZ }, - { "pagesz", BT_CONTROL_PAGESZ }, - { "mt", -1 }, - { "fastinsert", -2 }, - { 0, 0 } - }; - const char *z = zCfg; - int n = strlen(z); - char *aSpace; - const char *zOpt; - const char *zArg; - - aSpace = (char*)testMalloc(n+2); - while( rc==SQLITE4_OK && 0==testParseOption(&z, &zOpt, &zArg, aSpace) ){ - int i; - int iVal; - rc = testArgSelect(aParam, "param", zOpt, &i); - if( rc!=SQLITE4_OK ) break; - - rc = testParseInt(zArg, &iVal); - if( rc!=SQLITE4_OK ) break; - - switch( aParam[i].eParam ){ - case -1: - *pbMt = iVal; - break; - case -2: - pDb->bFastInsert = 1; - break; - default: - rc = sqlite4BtControl(pDb->pBt, aParam[i].eParam, (void*)&iVal); - break; - } - } - testFree(aSpace); - } - - return rc; -} - - -int test_bt_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - - static const DatabaseMethods SqlMethods = { - bt_close, - bt_write, - bt_delete, - bt_delete_range, - bt_fetch, - bt_scan, - bt_begin, - bt_commit, - bt_rollback - }; - BtDb *p = 0; - bt_db *pBt = 0; - int rc; - sqlite4_env *pEnv = sqlite4_env_default(); - - if( bClear && zFilename && zFilename[0] ){ - char *zLog = sqlite3_mprintf("%s-wal", zFilename); - unlink(zFilename); - unlink(zLog); - sqlite3_free(zLog); - } - - rc = sqlite4BtNew(pEnv, 0, &pBt); - if( rc==SQLITE4_OK ){ - int mt = 0; /* True for multi-threaded connection */ - - p = (BtDb*)testMalloc(sizeof(BtDb)); - p->base.pMethods = &SqlMethods; - p->pBt = pBt; - p->pEnv = pEnv; - p->nRef = 1; - - p->env.pVfsCtx = (void*)p; - p->env.xFullpath = btVfsFullpath; - p->env.xOpen = btVfsOpen; - p->env.xSize = btVfsSize; - p->env.xRead = btVfsRead; - p->env.xWrite = btVfsWrite; - p->env.xTruncate = btVfsTruncate; - p->env.xSync = btVfsSync; - p->env.xSectorSize = btVfsSectorSize; - p->env.xClose = btVfsClose; - p->env.xUnlink = btVfsUnlink; - p->env.xLock = btVfsLock; - p->env.xTestLock = btVfsTestLock; - p->env.xShmMap = btVfsShmMap; - p->env.xShmBarrier = btVfsShmBarrier; - p->env.xShmUnmap = btVfsShmUnmap; - - sqlite4BtControl(pBt, BT_CONTROL_GETVFS, (void*)&p->pVfs); - sqlite4BtControl(pBt, BT_CONTROL_SETVFS, (void*)&p->env); - - rc = testBtConfigure(p, zSpec, &mt); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtOpen(pBt, zFilename); - } - - if( rc==SQLITE4_OK && mt ){ - int nAuto = 0; - rc = bgc_attach(p, zSpec); - sqlite4BtControl(pBt, BT_CONTROL_AUTOCKPT, (void*)&nAuto); - } - } - - if( rc!=SQLITE4_OK && p ){ - bt_close(&p->base); - } - - *ppDb = &p->base; - return rc; -} - -int test_fbt_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return test_bt_open("fast=1", zFilename, bClear, ppDb); -} - -int test_fbts_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return test_bt_open("fast=1 blksz=32K pagesz=512", zFilename, bClear, ppDb); -} - - -void tdb_bt_prepare_sync_crash(TestDb *pTestDb, int iSync){ - BtDb *p = (BtDb*)pTestDb; - assert( pTestDb->pMethods->xClose==bt_close ); - assert( p->bCrash==0 ); - p->nCrashSync = iSync; -} - -bt_db *tdb_bt(TestDb *pDb){ - if( pDb->pMethods->xClose==bt_close ){ - return ((BtDb *)pDb)->pBt; - } - return 0; -} - -/************************************************************************* -** Beginning of code for background checkpointer. -*/ - -struct bt_ckpter { - sqlite4_buffer file; /* File name */ - sqlite4_buffer spec; /* Options */ - int nLogsize; /* Minimum log size to checkpoint */ - int nRef; /* Number of clients */ - - int bDoWork; /* Set by client threads */ - pthread_t ckpter_thread; /* Checkpointer thread */ - pthread_cond_t ckpter_cond; /* Condition var the ckpter waits on */ - pthread_mutex_t ckpter_mutex; /* Mutex used with ckpter_cond */ - - bt_ckpter *pNext; /* Next object in list at gBgc.pCkpter */ -}; - -static struct GlobalBackgroundCheckpointer { - bt_ckpter *pCkpter; /* Linked list of checkpointers */ -} gBgc; - -static void *bgc_main(void *pArg){ - BtDb *pDb = 0; - int rc; - int mt; - bt_ckpter *pCkpter = (bt_ckpter*)pArg; - - rc = test_bt_open("", (char*)pCkpter->file.p, 0, (TestDb**)&pDb); - assert( rc==SQLITE4_OK ); - rc = testBtConfigure(pDb, (char*)pCkpter->spec.p, &mt); - - while( pCkpter->nRef>0 ){ - bt_db *db = pDb->pBt; - int nLog = 0; - - sqlite4BtBegin(db, 1); - sqlite4BtCommit(db, 0); - sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog); - - if( nLog>=pCkpter->nLogsize ){ - int rc; - bt_checkpoint ckpt; - memset(&ckpt, 0, sizeof(bt_checkpoint)); - ckpt.nFrameBuffer = nLog/2; - rc = sqlite4BtControl(db, BT_CONTROL_CHECKPOINT, (void*)&ckpt); - assert( rc==SQLITE4_OK ); - sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog); - } - - /* The thread will wake up when it is signaled either because another - ** thread has created some work for this one or because the connection - ** is being closed. */ - pthread_mutex_lock(&pCkpter->ckpter_mutex); - if( pCkpter->bDoWork==0 ){ - pthread_cond_wait(&pCkpter->ckpter_cond, &pCkpter->ckpter_mutex); - } - pCkpter->bDoWork = 0; - pthread_mutex_unlock(&pCkpter->ckpter_mutex); - } - - if( pDb ) bt_close((TestDb*)pDb); - return 0; -} - -static void bgc_logsize_cb(void *pCtx, int nLogsize){ - bt_ckpter *p = (bt_ckpter*)pCtx; - if( nLogsize>=p->nLogsize ){ - pthread_mutex_lock(&p->ckpter_mutex); - p->bDoWork = 1; - pthread_cond_signal(&p->ckpter_cond); - pthread_mutex_unlock(&p->ckpter_mutex); - } -} - -static int bgc_attach(BtDb *pDb, const char *zSpec){ - int rc; - int n; - bt_info info; - bt_ckpter *pCkpter; - - /* Figure out the full path to the database opened by handle pDb. */ - info.eType = BT_INFO_FILENAME; - info.pgno = 0; - sqlite4_buffer_init(&info.output, 0); - rc = sqlite4BtControl(pDb->pBt, BT_CONTROL_INFO, (void*)&info); - if( rc!=SQLITE4_OK ) return rc; - - sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV)); - - /* Search for an existing bt_ckpter object. */ - n = info.output.n; - for(pCkpter=gBgc.pCkpter; pCkpter; pCkpter=pCkpter->pNext){ - if( n==pCkpter->file.n && 0==memcmp(info.output.p, pCkpter->file.p, n) ){ - break; - } - } - - /* Failed to find a suitable checkpointer. Create a new one. */ - if( pCkpter==0 ){ - bt_logsizecb cb; - - pCkpter = testMalloc(sizeof(bt_ckpter)); - memcpy(&pCkpter->file, &info.output, sizeof(sqlite4_buffer)); - info.output.p = 0; - pCkpter->pNext = gBgc.pCkpter; - pCkpter->nLogsize = 1000; - gBgc.pCkpter = pCkpter; - pCkpter->nRef = 1; - - sqlite4_buffer_init(&pCkpter->spec, 0); - rc = sqlite4_buffer_set(&pCkpter->spec, zSpec, strlen(zSpec)+1); - assert( rc==SQLITE4_OK ); - - /* Kick off the checkpointer thread. */ - if( rc==0 ) rc = pthread_cond_init(&pCkpter->ckpter_cond, 0); - if( rc==0 ) rc = pthread_mutex_init(&pCkpter->ckpter_mutex, 0); - if( rc==0 ){ - rc = pthread_create(&pCkpter->ckpter_thread, 0, bgc_main, (void*)pCkpter); - } - assert( rc==0 ); /* todo: Fix this */ - - /* Set up the logsize callback for the client thread */ - cb.pCtx = (void*)pCkpter; - cb.xLogsize = bgc_logsize_cb; - sqlite4BtControl(pDb->pBt, BT_CONTROL_LOGSIZECB, (void*)&cb); - }else{ - pCkpter->nRef++; - } - - /* Assuming a checkpointer was encountered or effected, attach the - ** connection to it. */ - if( pCkpter ){ - pDb->pCkpter = pCkpter; - } - - sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV)); - sqlite4_buffer_clear(&info.output); - return rc; -} - -static int bgc_detach(BtDb *pDb){ - int rc = SQLITE4_OK; - bt_ckpter *pCkpter = pDb->pCkpter; - if( pCkpter ){ - int bShutdown = 0; /* True if this is the last reference */ - - sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV)); - pCkpter->nRef--; - if( pCkpter->nRef==0 ){ - bt_ckpter **pp; - - *pp = pCkpter->pNext; - for(pp=&gBgc.pCkpter; *pp!=pCkpter; pp=&((*pp)->pNext)); - bShutdown = 1; - } - sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV)); - - if( bShutdown ){ - void *pDummy; - - /* Signal the checkpointer thread. */ - pthread_mutex_lock(&pCkpter->ckpter_mutex); - pCkpter->bDoWork = 1; - pthread_cond_signal(&pCkpter->ckpter_cond); - pthread_mutex_unlock(&pCkpter->ckpter_mutex); - - /* Join the checkpointer thread. */ - pthread_join(pCkpter->ckpter_thread, &pDummy); - pthread_cond_destroy(&pCkpter->ckpter_cond); - pthread_mutex_destroy(&pCkpter->ckpter_mutex); - - sqlite4_buffer_clear(&pCkpter->file); - sqlite4_buffer_clear(&pCkpter->spec); - testFree(pCkpter); - } - - pDb->pCkpter = 0; - } - return rc; -} - -/* -** End of background checkpointer. -*************************************************************************/ diff --git a/ext/lsm1/lsm-test/lsmtest_util.c b/ext/lsm1/lsm-test/lsmtest_util.c deleted file mode 100644 index adab8a53e..000000000 --- a/ext/lsm1/lsm-test/lsmtest_util.c +++ /dev/null @@ -1,223 +0,0 @@ - -#include "lsmtest.h" -#include <stdarg.h> -#include <stdio.h> -#include <string.h> -#ifndef _WIN32 -# include <sys/time.h> -#endif - -/* -** Global variables used within this module. -*/ -static struct TestutilGlobal { - char **argv; - int argc; -} g = {0, 0}; - -static struct TestutilRnd { - unsigned int aRand1[2048]; /* Bits 0..10 */ - unsigned int aRand2[2048]; /* Bits 11..21 */ - unsigned int aRand3[1024]; /* Bits 22..31 */ -} r; - -/************************************************************************* -** The following block is a copy of the implementation of SQLite function -** sqlite3_randomness. This version has two important differences: -** -** 1. It always uses the same seed. So the sequence of random data output -** is the same for every run of the program. -** -** 2. It is not threadsafe. -*/ -static struct sqlite3PrngType { - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ -} sqlite3Prng = { - 0xAF, 0x28, - { - 0x71, 0xF5, 0xB4, 0x6E, 0x80, 0xAB, 0x1D, 0xB8, - 0xFB, 0xB7, 0x49, 0xBF, 0xFF, 0x72, 0x2D, 0x14, - 0x79, 0x09, 0xE3, 0x78, 0x76, 0xB0, 0x2C, 0x0A, - 0x8E, 0x23, 0xEE, 0xDF, 0xE0, 0x9A, 0x2F, 0x67, - 0xE1, 0xBE, 0x0E, 0xA7, 0x08, 0x97, 0xEB, 0x77, - 0x78, 0xBA, 0x9D, 0xCA, 0x49, 0x4C, 0x60, 0x9A, - 0xF6, 0xBD, 0xDA, 0x7F, 0xBC, 0x48, 0x58, 0x52, - 0xE5, 0xCD, 0x83, 0x72, 0x23, 0x52, 0xFF, 0x6D, - 0xEF, 0x0F, 0x82, 0x29, 0xA0, 0x83, 0x3F, 0x7D, - 0xA4, 0x88, 0x31, 0xE7, 0x88, 0x92, 0x3B, 0x9B, - 0x3B, 0x2C, 0xC2, 0x4C, 0x71, 0xA2, 0xB0, 0xEA, - 0x36, 0xD0, 0x00, 0xF1, 0xD3, 0x39, 0x17, 0x5D, - 0x2A, 0x7A, 0xE4, 0xAD, 0xE1, 0x64, 0xCE, 0x0F, - 0x9C, 0xD9, 0xF5, 0xED, 0xB0, 0x22, 0x5E, 0x62, - 0x97, 0x02, 0xA3, 0x8C, 0x67, 0x80, 0xFC, 0x88, - 0x14, 0x0B, 0x15, 0x10, 0x0F, 0xC7, 0x40, 0xD4, - 0xF1, 0xF9, 0x0E, 0x1A, 0xCE, 0xB9, 0x1E, 0xA1, - 0x72, 0x8E, 0xD7, 0x78, 0x39, 0xCD, 0xF4, 0x5D, - 0x2A, 0x59, 0x26, 0x34, 0xF2, 0x73, 0x0B, 0xA0, - 0x02, 0x51, 0x2C, 0x03, 0xA3, 0xA7, 0x43, 0x13, - 0xE8, 0x98, 0x2B, 0xD2, 0x53, 0xF8, 0xEE, 0x91, - 0x7D, 0xE7, 0xE3, 0xDA, 0xD5, 0xBB, 0xC0, 0x92, - 0x9D, 0x98, 0x01, 0x2C, 0xF9, 0xB9, 0xA0, 0xEB, - 0xCF, 0x32, 0xFA, 0x01, 0x49, 0xA5, 0x1D, 0x9A, - 0x76, 0x86, 0x3F, 0x40, 0xD4, 0x89, 0x8F, 0x9C, - 0xE2, 0xE3, 0x11, 0x31, 0x37, 0xB2, 0x49, 0x28, - 0x35, 0xC0, 0x99, 0xB6, 0xD0, 0xBC, 0x66, 0x35, - 0xF7, 0x83, 0x5B, 0xD7, 0x37, 0x1A, 0x2B, 0x18, - 0xA6, 0xFF, 0x8D, 0x7C, 0x81, 0xA8, 0xFC, 0x9E, - 0xC4, 0xEC, 0x80, 0xD0, 0x98, 0xA7, 0x76, 0xCC, - 0x9C, 0x2F, 0x7B, 0xFF, 0x8E, 0x0E, 0xBB, 0x90, - 0xAE, 0x13, 0x06, 0xF5, 0x1C, 0x4E, 0x52, 0xF7 - } -}; - -/* Generate and return single random byte */ -static unsigned char randomByte(void){ - unsigned char t; - sqlite3Prng.i++; - t = sqlite3Prng.s[sqlite3Prng.i]; - sqlite3Prng.j += t; - sqlite3Prng.s[sqlite3Prng.i] = sqlite3Prng.s[sqlite3Prng.j]; - sqlite3Prng.s[sqlite3Prng.j] = t; - t += sqlite3Prng.s[sqlite3Prng.i]; - return sqlite3Prng.s[t]; -} - -/* -** Return N random bytes. -*/ -static void randomBlob(int nBuf, unsigned char *zBuf){ - int i; - for(i=0; i<nBuf; i++){ - zBuf[i] = randomByte(); - } -} -/* -** End of code copied from SQLite. -*************************************************************************/ - - -int testPrngInit(void){ - randomBlob(sizeof(r.aRand1), (unsigned char *)r.aRand1); - randomBlob(sizeof(r.aRand2), (unsigned char *)r.aRand2); - randomBlob(sizeof(r.aRand3), (unsigned char *)r.aRand3); - return 0; -} - -unsigned int testPrngValue(unsigned int iVal){ - return - r.aRand1[iVal & 0x000007FF] ^ - r.aRand2[(iVal>>11) & 0x000007FF] ^ - r.aRand3[(iVal>>22) & 0x000003FF] - ; -} - -void testPrngArray(unsigned int iVal, unsigned int *aOut, int nOut){ - int i; - for(i=0; i<nOut; i++){ - aOut[i] = testPrngValue(iVal+i); - } -} - -void testPrngString(unsigned int iVal, char *aOut, int nOut){ - int i; - for(i=0; i<(nOut-1); i++){ - aOut[i] = 'a' + (testPrngValue(iVal+i) % 26); - } - aOut[i] = '\0'; -} - -void testErrorInit(int argc, char **argv){ - g.argc = argc; - g.argv = argv; -} - -void testPrintError(const char *zFormat, ...){ - va_list ap; - va_start(ap, zFormat); - vfprintf(stderr, zFormat, ap); - va_end(ap); -} - -void testPrintFUsage(const char *zFormat, ...){ - va_list ap; - va_start(ap, zFormat); - fprintf(stderr, "Usage: %s %s ", g.argv[0], g.argv[1]); - vfprintf(stderr, zFormat, ap); - fprintf(stderr, "\n"); - va_end(ap); -} - -void testPrintUsage(const char *zArgs){ - testPrintError("Usage: %s %s %s\n", g.argv[0], g.argv[1], zArgs); -} - - -static void argError(void *aData, const char *zType, int sz, const char *zArg){ - struct Entry { const char *zName; }; - struct Entry *pEntry; - const char *zPrev = 0; - - testPrintError("unrecognized %s \"%s\": must be ", zType, zArg); - for(pEntry=(struct Entry *)aData; - pEntry->zName; - pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] - ){ - if( zPrev ){ testPrintError("%s, ", zPrev); } - zPrev = pEntry->zName; - } - testPrintError("or %s\n", zPrev); -} - -int testArgSelectX( - void *aData, - const char *zType, - int sz, - const char *zArg, - int *piOut -){ - struct Entry { const char *zName; }; - struct Entry *pEntry; - int nArg = strlen(zArg); - - int i = 0; - int iOut = -1; - int nOut = 0; - - for(pEntry=(struct Entry *)aData; - pEntry->zName; - pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] - ){ - int nName = strlen(pEntry->zName); - if( nArg<=nName && memcmp(pEntry->zName, zArg, nArg)==0 ){ - iOut = i; - if( nName==nArg ){ - nOut = 1; - break; - } - nOut++; - } - i++; - } - - if( nOut!=1 ){ - argError(aData, zType, sz, zArg); - }else{ - *piOut = iOut; - } - return (nOut!=1); -} - -struct timeval zero_time; - -void testTimeInit(void){ - gettimeofday(&zero_time, 0); -} - -int testTimeGet(void){ - struct timeval now; - gettimeofday(&now, 0); - return - (((int)now.tv_sec - (int)zero_time.tv_sec)*1000) + - (((int)now.tv_usec - (int)zero_time.tv_usec)/1000); -} diff --git a/ext/lsm1/lsm-test/lsmtest_win32.c b/ext/lsm1/lsm-test/lsmtest_win32.c deleted file mode 100644 index 947272336..000000000 --- a/ext/lsm1/lsm-test/lsmtest_win32.c +++ /dev/null @@ -1,30 +0,0 @@ - -#include "lsmtest.h" - -#ifdef _WIN32 - -#define TICKS_PER_SECOND (10000000) -#define TICKS_PER_MICROSECOND (10) -#define TICKS_UNIX_EPOCH (116444736000000000LL) - -int win32GetTimeOfDay( - struct timeval *tp, - void *tzp -){ - FILETIME fileTime; - ULONGLONG ticks; - ULONGLONG unixTicks; - - unused_parameter(tzp); - memset(&fileTime, 0, sizeof(FILETIME)); - GetSystemTimeAsFileTime(&fileTime); - ticks = (ULONGLONG)fileTime.dwHighDateTime << 32; - ticks |= (ULONGLONG)fileTime.dwLowDateTime; - unixTicks = ticks - TICKS_UNIX_EPOCH; - tp->tv_sec = (long)(unixTicks / TICKS_PER_SECOND); - unixTicks -= ((ULONGLONG)tp->tv_sec * TICKS_PER_SECOND); - tp->tv_usec = (long)(unixTicks / TICKS_PER_MICROSECOND); - - return 0; -} -#endif diff --git a/ext/lsm1/lsm.h b/ext/lsm1/lsm.h deleted file mode 100644 index 48701c4c5..000000000 --- a/ext/lsm1/lsm.h +++ /dev/null @@ -1,684 +0,0 @@ -/* -** 2011-08-10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file defines the LSM API. -*/ -#ifndef _LSM_H -#define _LSM_H -#include <stddef.h> -#ifdef __cplusplus -extern "C" { -#endif - -/* -** Opaque handle types. -*/ -typedef struct lsm_compress lsm_compress; /* Compression library functions */ -typedef struct lsm_compress_factory lsm_compress_factory; -typedef struct lsm_cursor lsm_cursor; /* Database cursor handle */ -typedef struct lsm_db lsm_db; /* Database connection handle */ -typedef struct lsm_env lsm_env; /* Runtime environment */ -typedef struct lsm_file lsm_file; /* OS file handle */ -typedef struct lsm_mutex lsm_mutex; /* Mutex handle */ - -/* 64-bit integer type used for file offsets. */ -typedef long long int lsm_i64; /* 64-bit signed integer type */ - -/* Candidate values for the 3rd argument to lsm_env.xLock() */ -#define LSM_LOCK_UNLOCK 0 -#define LSM_LOCK_SHARED 1 -#define LSM_LOCK_EXCL 2 - -/* Flags for lsm_env.xOpen() */ -#define LSM_OPEN_READONLY 0x0001 - -/* -** CAPI: Database Runtime Environment -** -** Run-time environment used by LSM -*/ -struct lsm_env { - int nByte; /* Size of this structure in bytes */ - int iVersion; /* Version number of this structure (1) */ - /****** file i/o ***********************************************/ - void *pVfsCtx; - int (*xFullpath)(lsm_env*, const char *, char *, int *); - int (*xOpen)(lsm_env*, const char *, int flags, lsm_file **); - int (*xRead)(lsm_file *, lsm_i64, void *, int); - int (*xWrite)(lsm_file *, lsm_i64, void *, int); - int (*xTruncate)(lsm_file *, lsm_i64); - int (*xSync)(lsm_file *); - int (*xSectorSize)(lsm_file *); - int (*xRemap)(lsm_file *, lsm_i64, void **, lsm_i64*); - int (*xFileid)(lsm_file *, void *pBuf, int *pnBuf); - int (*xClose)(lsm_file *); - int (*xUnlink)(lsm_env*, const char *); - int (*xLock)(lsm_file*, int, int); - int (*xTestLock)(lsm_file*, int, int, int); - int (*xShmMap)(lsm_file*, int, int, void **); - void (*xShmBarrier)(void); - int (*xShmUnmap)(lsm_file*, int); - /****** memory allocation ****************************************/ - void *pMemCtx; - void *(*xMalloc)(lsm_env*, size_t); /* malloc(3) function */ - void *(*xRealloc)(lsm_env*, void *, size_t); /* realloc(3) function */ - void (*xFree)(lsm_env*, void *); /* free(3) function */ - size_t (*xSize)(lsm_env*, void *); /* xSize function */ - /****** mutexes ****************************************************/ - void *pMutexCtx; - int (*xMutexStatic)(lsm_env*,int,lsm_mutex**); /* Obtain a static mutex */ - int (*xMutexNew)(lsm_env*, lsm_mutex**); /* Get a new dynamic mutex */ - void (*xMutexDel)(lsm_mutex *); /* Delete an allocated mutex */ - void (*xMutexEnter)(lsm_mutex *); /* Grab a mutex */ - int (*xMutexTry)(lsm_mutex *); /* Attempt to obtain a mutex */ - void (*xMutexLeave)(lsm_mutex *); /* Leave a mutex */ - int (*xMutexHeld)(lsm_mutex *); /* Return true if mutex is held */ - int (*xMutexNotHeld)(lsm_mutex *); /* Return true if mutex not held */ - /****** other ****************************************************/ - int (*xSleep)(lsm_env*, int microseconds); - - /* New fields may be added in future releases, in which case the - ** iVersion value will increase. */ -}; - -/* -** Values that may be passed as the second argument to xMutexStatic. -*/ -#define LSM_MUTEX_GLOBAL 1 -#define LSM_MUTEX_HEAP 2 - -/* -** CAPI: LSM Error Codes -*/ -#define LSM_OK 0 -#define LSM_ERROR 1 -#define LSM_BUSY 5 -#define LSM_NOMEM 7 -#define LSM_READONLY 8 -#define LSM_IOERR 10 -#define LSM_CORRUPT 11 -#define LSM_FULL 13 -#define LSM_CANTOPEN 14 -#define LSM_PROTOCOL 15 -#define LSM_MISUSE 21 - -#define LSM_MISMATCH 50 - - -#define LSM_IOERR_NOENT (LSM_IOERR | (1<<8)) - -/* -** CAPI: Creating and Destroying Database Connection Handles -** -** Open and close a database connection handle. -*/ -int lsm_new(lsm_env*, lsm_db **ppDb); -int lsm_close(lsm_db *pDb); - -/* -** CAPI: Connecting to a Database -*/ -int lsm_open(lsm_db *pDb, const char *zFilename); - -/* -** CAPI: Obtaining pointers to database environments -** -** Return a pointer to the environment used by the database connection -** passed as the first argument. Assuming the argument is valid, this -** function always returns a valid environment pointer - it cannot fail. -*/ -lsm_env *lsm_get_env(lsm_db *pDb); - -/* -** The lsm_default_env() function returns a pointer to the default LSM -** environment for the current platform. -*/ -lsm_env *lsm_default_env(void); - - -/* -** CAPI: Configuring a database connection. -** -** The lsm_config() function is used to configure a database connection. -*/ -int lsm_config(lsm_db *, int, ...); - -/* -** The following values may be passed as the second argument to lsm_config(). -** -** LSM_CONFIG_AUTOFLUSH: -** A read/write integer parameter. -** -** This value determines the amount of data allowed to accumulate in a -** live in-memory tree before it is marked as old. After committing a -** transaction, a connection checks if the size of the live in-memory tree, -** including data structure overhead, is greater than the value of this -** option in KB. If it is, and there is not already an old in-memory tree, -** the live in-memory tree is marked as old. -** -** The maximum allowable value is 1048576 (1GB). There is no minimum -** value. If this parameter is set to zero, then an attempt is made to -** mark the live in-memory tree as old after each transaction is committed. -** -** The default value is 1024 (1MB). -** -** LSM_CONFIG_PAGE_SIZE: -** A read/write integer parameter. This parameter may only be set before -** lsm_open() has been called. -** -** LSM_CONFIG_BLOCK_SIZE: -** A read/write integer parameter. -** -** This parameter may only be set before lsm_open() has been called. It -** must be set to a power of two between 64 and 65536, inclusive (block -** sizes between 64KB and 64MB). -** -** If the connection creates a new database, the block size of the new -** database is set to the value of this option in KB. After lsm_open() -** has been called, querying this parameter returns the actual block -** size of the opened database. -** -** The default value is 1024 (1MB blocks). -** -** LSM_CONFIG_SAFETY: -** A read/write integer parameter. Valid values are 0, 1 (the default) -** and 2. This parameter determines how robust the database is in the -** face of a system crash (e.g. a power failure or operating system -** crash). As follows: -** -** 0 (off): No robustness. A system crash may corrupt the database. -** -** 1 (normal): Some robustness. A system crash may not corrupt the -** database file, but recently committed transactions may -** be lost following recovery. -** -** 2 (full): Full robustness. A system crash may not corrupt the -** database file. Following recovery the database file -** contains all successfully committed transactions. -** -** LSM_CONFIG_AUTOWORK: -** A read/write integer parameter. -** -** LSM_CONFIG_AUTOCHECKPOINT: -** A read/write integer parameter. -** -** If this option is set to non-zero value N, then a checkpoint is -** automatically attempted after each N KB of data have been written to -** the database file. -** -** The amount of uncheckpointed data already written to the database file -** is a global parameter. After performing database work (writing to the -** database file), the process checks if the total amount of uncheckpointed -** data exceeds the value of this paramter. If so, a checkpoint is performed. -** This means that this option may cause the connection to perform a -** checkpoint even if the current connection has itself written very little -** data into the database file. -** -** The default value is 2048 (checkpoint every 2MB). -** -** LSM_CONFIG_MMAP: -** A read/write integer parameter. If this value is set to 0, then the -** database file is accessed using ordinary read/write IO functions. Or, -** if it is set to 1, then the database file is memory mapped and accessed -** that way. If this parameter is set to any value N greater than 1, then -** up to the first N KB of the file are memory mapped, and any remainder -** accessed using read/write IO. -** -** The default value is 1 on 64-bit platforms and 32768 on 32-bit platforms. -** -** -** LSM_CONFIG_USE_LOG: -** A read/write boolean parameter. True (the default) to use the log -** file normally. False otherwise. -** -** LSM_CONFIG_AUTOMERGE: -** A read/write integer parameter. The minimum number of segments to -** merge together at a time. Default value 4. -** -** LSM_CONFIG_MAX_FREELIST: -** A read/write integer parameter. The maximum number of free-list -** entries that are stored in a database checkpoint (the others are -** stored elsewhere in the database). -** -** There is no reason for an application to configure or query this -** parameter. It is only present because configuring a small value -** makes certain parts of the lsm code easier to test. -** -** LSM_CONFIG_MULTIPLE_PROCESSES: -** A read/write boolean parameter. This parameter may only be set before -** lsm_open() has been called. If true, the library uses shared-memory -** and posix advisory locks to co-ordinate access by clients from within -** multiple processes. Otherwise, if false, all database clients must be -** located in the same process. The default value is true. -** -** LSM_CONFIG_SET_COMPRESSION: -** Set the compression methods used to compress and decompress database -** content. The argument to this option should be a pointer to a structure -** of type lsm_compress. The lsm_config() method takes a copy of the -** structures contents. -** -** This option may only be used before lsm_open() is called. Invoking it -** after lsm_open() has been called results in an LSM_MISUSE error. -** -** LSM_CONFIG_GET_COMPRESSION: -** Query the compression methods used to compress and decompress database -** content. -** -** LSM_CONFIG_SET_COMPRESSION_FACTORY: -** Configure a factory method to be invoked in case of an LSM_MISMATCH -** error. -** -** LSM_CONFIG_READONLY: -** A read/write boolean parameter. This parameter may only be set before -** lsm_open() is called. -*/ -#define LSM_CONFIG_AUTOFLUSH 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_AUTOMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 -#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 -#define LSM_CONFIG_READONLY 16 - -#define LSM_SAFETY_OFF 0 -#define LSM_SAFETY_NORMAL 1 -#define LSM_SAFETY_FULL 2 - -/* -** CAPI: Compression and/or Encryption Hooks -*/ -struct lsm_compress { - void *pCtx; - unsigned int iId; - int (*xBound)(void *, int nSrc); - int (*xCompress)(void *, char *, int *, const char *, int); - int (*xUncompress)(void *, char *, int *, const char *, int); - void (*xFree)(void *pCtx); -}; - -struct lsm_compress_factory { - void *pCtx; - int (*xFactory)(void *, lsm_db *, unsigned int); - void (*xFree)(void *pCtx); -}; - -#define LSM_COMPRESSION_EMPTY 0 -#define LSM_COMPRESSION_NONE 1 - -/* -** CAPI: Allocating and Freeing Memory -** -** Invoke the memory allocation functions that belong to environment -** pEnv. Or the system defaults if no memory allocation functions have -** been registered. -*/ -void *lsm_malloc(lsm_env*, size_t); -void *lsm_realloc(lsm_env*, void *, size_t); -void lsm_free(lsm_env*, void *); - -/* -** CAPI: Querying a Connection For Operational Data -** -** Query a database connection for operational statistics or data. -*/ -int lsm_info(lsm_db *, int, ...); - -int lsm_get_user_version(lsm_db *, unsigned int *); -int lsm_set_user_version(lsm_db *, unsigned int); - -/* -** The following values may be passed as the second argument to lsm_info(). -** -** LSM_INFO_NWRITE: -** The third parameter should be of type (int *). The location pointed -** to by the third parameter is set to the number of 4KB pages written to -** the database file during the lifetime of this connection. -** -** LSM_INFO_NREAD: -** The third parameter should be of type (int *). The location pointed -** to by the third parameter is set to the number of 4KB pages read from -** the database file during the lifetime of this connection. -** -** LSM_INFO_DB_STRUCTURE: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure reflecting the -** current structure of the database file. Specifically, the current state -** of the worker snapshot. The returned string should be eventually freed -** by the caller using lsm_free(). -** -** The returned list contains one element for each level in the database, -** in order from most to least recent. Each element contains a -** single element for each segment comprising the corresponding level, -** starting with the lhs segment, then each of the rhs segments (if any) -** in order from most to least recent. -** -** Each segment element is itself a list of 4 integer values, as follows: -** -** <ol><li> First page of segment -** <li> Last page of segment -** <li> Root page of segment (if applicable) -** <li> Total number of pages in segment -** </ol> -** -** LSM_INFO_ARRAY_STRUCTURE: -** There should be two arguments passed following this option (i.e. a -** total of four arguments passed to lsm_info()). The first argument -** should be the page number of the first page in a database array -** (perhaps obtained from an earlier INFO_DB_STRUCTURE call). The second -** trailing argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string that must -** be eventually freed using lsm_free() by the caller. -** -** The output string contains the text representation of a Tcl list of -** integers. Each pair of integers represent a range of pages used by -** the identified array. For example, if the array occupies database -** pages 993 to 1024, then pages 2048 to 2777, then the returned string -** will be "993 1024 2048 2777". -** -** If the specified integer argument does not correspond to the first -** page of any database array, LSM_ERROR is returned and the output -** pointer is set to a NULL value. -** -** LSM_INFO_LOG_STRUCTURE: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure. The returned -** string should be eventually freed by the caller using lsm_free(). -** -** The Tcl structure returned is a list of six integers that describe -** the current structure of the log file. -** -** LSM_INFO_ARRAY_PAGES: -** -** LSM_INFO_PAGE_ASCII_DUMP: -** As with LSM_INFO_ARRAY_STRUCTURE, there should be two arguments passed -** with calls that specify this option - an integer page number and a -** (char **) used to return a nul-terminated string that must be later -** freed using lsm_free(). In this case the output string is populated -** with a human-readable description of the page content. -** -** If the page cannot be decoded, it is not an error. In this case the -** human-readable output message will report the systems failure to -** interpret the page data. -** -** LSM_INFO_PAGE_HEX_DUMP: -** This argument is similar to PAGE_ASCII_DUMP, except that keys and -** values are represented using hexadecimal notation instead of ascii. -** -** LSM_INFO_FREELIST: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure. The returned -** string should be eventually freed by the caller using lsm_free(). -** -** The Tcl structure returned is a list containing one element for each -** free block in the database. The element itself consists of two -** integers - the block number and the id of the snapshot that freed it. -** -** LSM_INFO_CHECKPOINT_SIZE: -** The third argument should be of type (int *). The location pointed to -** by this argument is populated with the number of KB written to the -** database file since the most recent checkpoint. -** -** LSM_INFO_TREE_SIZE: -** If this value is passed as the second argument to an lsm_info() call, it -** should be followed by two arguments of type (int *) (for a total of four -** arguments). -** -** At any time, there are either one or two tree structures held in shared -** memory that new database clients will access (there may also be additional -** tree structures being used by older clients - this API does not provide -** information on them). One tree structure - the current tree - is used to -** accumulate new data written to the database. The other tree structure - -** the old tree - is a read-only tree holding older data and may be flushed -** to disk at any time. -** -** Assuming no error occurs, the location pointed to by the first of the two -** (int *) arguments is set to the size of the old in-memory tree in KB. -** The second is set to the size of the current, or live in-memory tree. -** -** LSM_INFO_COMPRESSION_ID: -** This value should be followed by a single argument of type -** (unsigned int *). If successful, the location pointed to is populated -** with the database compression id before returning. -*/ -#define LSM_INFO_NWRITE 1 -#define LSM_INFO_NREAD 2 -#define LSM_INFO_DB_STRUCTURE 3 -#define LSM_INFO_LOG_STRUCTURE 4 -#define LSM_INFO_ARRAY_STRUCTURE 5 -#define LSM_INFO_PAGE_ASCII_DUMP 6 -#define LSM_INFO_PAGE_HEX_DUMP 7 -#define LSM_INFO_FREELIST 8 -#define LSM_INFO_ARRAY_PAGES 9 -#define LSM_INFO_CHECKPOINT_SIZE 10 -#define LSM_INFO_TREE_SIZE 11 -#define LSM_INFO_FREELIST_SIZE 12 -#define LSM_INFO_COMPRESSION_ID 13 - - -/* -** CAPI: Opening and Closing Write Transactions -** -** These functions are used to open and close transactions and nested -** sub-transactions. -** -** The lsm_begin() function is used to open transactions and sub-transactions. -** A successful call to lsm_begin() ensures that there are at least iLevel -** nested transactions open. To open a top-level transaction, pass iLevel=1. -** To open a sub-transaction within the top-level transaction, iLevel=2. -** Passing iLevel=0 is a no-op. -** -** lsm_commit() is used to commit transactions and sub-transactions. A -** successful call to lsm_commit() ensures that there are at most iLevel -** nested transactions open. To commit a top-level transaction, pass iLevel=0. -** To commit all sub-transactions inside the main transaction, pass iLevel=1. -** -** Function lsm_rollback() is used to roll back transactions and -** sub-transactions. A successful call to lsm_rollback() restores the database -** to the state it was in when the iLevel'th nested sub-transaction (if any) -** was first opened. And then closes transactions to ensure that there are -** at most iLevel nested transactions open. Passing iLevel=0 rolls back and -** closes the top-level transaction. iLevel=1 also rolls back the top-level -** transaction, but leaves it open. iLevel=2 rolls back the sub-transaction -** nested directly inside the top-level transaction (and leaves it open). -*/ -int lsm_begin(lsm_db *pDb, int iLevel); -int lsm_commit(lsm_db *pDb, int iLevel); -int lsm_rollback(lsm_db *pDb, int iLevel); - -/* -** CAPI: Writing to a Database -** -** Write a new value into the database. If a value with a duplicate key -** already exists it is replaced. -*/ -int lsm_insert(lsm_db*, const void *pKey, int nKey, const void *pVal, int nVal); - -/* -** Delete a value from the database. No error is returned if the specified -** key value does not exist in the database. -*/ -int lsm_delete(lsm_db *, const void *pKey, int nKey); - -/* -** Delete all database entries with keys that are greater than (pKey1/nKey1) -** and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and -** (pKey2/nKey2) themselves, if they exist in the database, are not deleted. -** -** Return LSM_OK if successful, or an LSM error code otherwise. -*/ -int lsm_delete_range(lsm_db *, - const void *pKey1, int nKey1, const void *pKey2, int nKey2 -); - -/* -** CAPI: Explicit Database Work and Checkpointing -** -** This function is called by a thread to work on the database structure. -*/ -int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); - -int lsm_flush(lsm_db *pDb); - -/* -** Attempt to checkpoint the current database snapshot. Return an LSM -** error code if an error occurs or LSM_OK otherwise. -** -** If the current snapshot has already been checkpointed, calling this -** function is a no-op. In this case if pnKB is not NULL, *pnKB is -** set to 0. Or, if the current snapshot is successfully checkpointed -** by this function and pbKB is not NULL, *pnKB is set to the number -** of bytes written to the database file since the previous checkpoint -** (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query). -*/ -int lsm_checkpoint(lsm_db *pDb, int *pnKB); - -/* -** CAPI: Opening and Closing Database Cursors -** -** Open and close a database cursor. -*/ -int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); -int lsm_csr_close(lsm_cursor *pCsr); - -/* -** CAPI: Positioning Database Cursors -** -** If the fourth parameter is LSM_SEEK_EQ, LSM_SEEK_GE or LSM_SEEK_LE, -** this function searches the database for an entry with key (pKey/nKey). -** If an error occurs, an LSM error code is returned. Otherwise, LSM_OK. -** -** If no error occurs and the requested key is present in the database, the -** cursor is left pointing to the entry with the specified key. Or, if the -** specified key is not present in the database the state of the cursor -** depends on the value passed as the final parameter, as follows: -** -** LSM_SEEK_EQ: -** The cursor is left at EOF (invalidated). A call to lsm_csr_valid() -** returns non-zero. -** -** LSM_SEEK_LE: -** The cursor is left pointing to the largest key in the database that -** is smaller than (pKey/nKey). If the database contains no keys smaller -** than (pKey/nKey), the cursor is left at EOF. -** -** LSM_SEEK_GE: -** The cursor is left pointing to the smallest key in the database that -** is larger than (pKey/nKey). If the database contains no keys larger -** than (pKey/nKey), the cursor is left at EOF. -** -** If the fourth parameter is LSM_SEEK_LEFAST, this function searches the -** database in a similar manner to LSM_SEEK_LE, with two differences: -** -** <ol><li>Even if a key can be found (the cursor is not left at EOF), the -** lsm_csr_value() function may not be used (attempts to do so return -** LSM_MISUSE). -** -** <li>The key that the cursor is left pointing to may be one that has -** been recently deleted from the database. In this case it is -** guaranteed that the returned key is larger than any key currently -** in the database that is less than or equal to (pKey/nKey). -** </ol> -** -** LSM_SEEK_LEFAST requests are intended to be used to allocate database -** keys. -*/ -int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek); - -int lsm_csr_first(lsm_cursor *pCsr); -int lsm_csr_last(lsm_cursor *pCsr); - -/* -** Advance the specified cursor to the next or previous key in the database. -** Return LSM_OK if successful, or an LSM error code otherwise. -** -** Functions lsm_csr_seek(), lsm_csr_first() and lsm_csr_last() are "seek" -** functions. Whether or not lsm_csr_next and lsm_csr_prev may be called -** successfully also depends on the most recent seek function called on -** the cursor. Specifically: -** -** <ul> -** <li> At least one seek function must have been called on the cursor. -** <li> To call lsm_csr_next(), the most recent call to a seek function must -** have been either lsm_csr_first() or a call to lsm_csr_seek() specifying -** LSM_SEEK_GE. -** <li> To call lsm_csr_prev(), the most recent call to a seek function must -** have been either lsm_csr_last() or a call to lsm_csr_seek() specifying -** LSM_SEEK_LE. -** </ul> -** -** Otherwise, if the above conditions are not met when lsm_csr_next or -** lsm_csr_prev is called, LSM_MISUSE is returned and the cursor position -** remains unchanged. -*/ -int lsm_csr_next(lsm_cursor *pCsr); -int lsm_csr_prev(lsm_cursor *pCsr); - -/* -** Values that may be passed as the fourth argument to lsm_csr_seek(). -*/ -#define LSM_SEEK_LEFAST -2 -#define LSM_SEEK_LE -1 -#define LSM_SEEK_EQ 0 -#define LSM_SEEK_GE 1 - -/* -** CAPI: Extracting Data From Database Cursors -** -** Retrieve data from a database cursor. -*/ -int lsm_csr_valid(lsm_cursor *pCsr); -int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey); -int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal); - -/* -** If no error occurs, this function compares the database key passed via -** the pKey/nKey arguments with the key that the cursor passed as the first -** argument currently points to. If the cursors key is less than, equal to -** or greater than pKey/nKey, *piRes is set to less than, equal to or greater -** than zero before returning. LSM_OK is returned in this case. -** -** Or, if an error occurs, an LSM error code is returned and the final -** value of *piRes is undefined. If the cursor does not point to a valid -** key when this function is called, LSM_MISUSE is returned. -*/ -int lsm_csr_cmp(lsm_cursor *pCsr, const void *pKey, int nKey, int *piRes); - -/* -** CAPI: Change these!! -** -** Configure a callback to which debugging and other messages should -** be directed. Only useful for debugging lsm. -*/ -void lsm_config_log(lsm_db *, void (*)(void *, int, const char *), void *); - -/* -** Configure a callback that is invoked if the database connection ever -** writes to the database file. -*/ -void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); - -/* ENDOFAPI */ -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif -#endif /* ifndef _LSM_H */ diff --git a/ext/lsm1/lsmInt.h b/ext/lsm1/lsmInt.h deleted file mode 100644 index 4e3c5e59c..000000000 --- a/ext/lsm1/lsmInt.h +++ /dev/null @@ -1,997 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Internal structure definitions for the LSM module. -*/ -#ifndef _LSM_INT_H -#define _LSM_INT_H - -#include "lsm.h" -#include <assert.h> -#include <string.h> - -#include <stdarg.h> -#include <stdlib.h> -#include <stdio.h> -#include <ctype.h> - -#ifdef _WIN32 -# ifdef _MSC_VER -# define snprintf _snprintf -# endif -#else -# include <unistd.h> -#endif - -#ifdef NDEBUG -# ifdef LSM_DEBUG_EXPENSIVE -# undef LSM_DEBUG_EXPENSIVE -# endif -# ifdef LSM_DEBUG -# undef LSM_DEBUG -# endif -#else -# ifndef LSM_DEBUG -# define LSM_DEBUG -# endif -#endif - -/* #define LSM_DEBUG_EXPENSIVE 1 */ - -/* -** Default values for various data structure parameters. These may be -** overridden by calls to lsm_config(). -*/ -#define LSM_DFLT_PAGE_SIZE (4 * 1024) -#define LSM_DFLT_BLOCK_SIZE (1 * 1024 * 1024) -#define LSM_DFLT_AUTOFLUSH (1 * 1024 * 1024) -#define LSM_DFLT_AUTOCHECKPOINT (i64)(2 * 1024 * 1024) -#define LSM_DFLT_AUTOWORK 1 -#define LSM_DFLT_LOG_SIZE (128*1024) -#define LSM_DFLT_AUTOMERGE 4 -#define LSM_DFLT_SAFETY LSM_SAFETY_NORMAL -#define LSM_DFLT_MMAP (LSM_IS_64_BIT ? 1 : 32768) -#define LSM_DFLT_MULTIPLE_PROCESSES 1 -#define LSM_DFLT_USE_LOG 1 - -/* Initial values for log file checksums. These are only used if the -** database file does not contain a valid checkpoint. */ -#define LSM_CKSUM0_INIT 42 -#define LSM_CKSUM1_INIT 42 - -/* "mmap" mode is currently only used in environments with 64-bit address -** spaces. The following macro is used to test for this. */ -#define LSM_IS_64_BIT (sizeof(void*)==8) - -#define LSM_AUTOWORK_QUANT 32 - -typedef struct Database Database; -typedef struct DbLog DbLog; -typedef struct FileSystem FileSystem; -typedef struct Freelist Freelist; -typedef struct FreelistEntry FreelistEntry; -typedef struct Level Level; -typedef struct LogMark LogMark; -typedef struct LogRegion LogRegion; -typedef struct LogWriter LogWriter; -typedef struct LsmString LsmString; -typedef struct Mempool Mempool; -typedef struct Merge Merge; -typedef struct MergeInput MergeInput; -typedef struct MetaPage MetaPage; -typedef struct MultiCursor MultiCursor; -typedef struct Page Page; -typedef struct Redirect Redirect; -typedef struct Segment Segment; -typedef struct SegmentMerger SegmentMerger; -typedef struct ShmChunk ShmChunk; -typedef struct ShmHeader ShmHeader; -typedef struct ShmReader ShmReader; -typedef struct Snapshot Snapshot; -typedef struct TransMark TransMark; -typedef struct Tree Tree; -typedef struct TreeCursor TreeCursor; -typedef struct TreeHeader TreeHeader; -typedef struct TreeMark TreeMark; -typedef struct TreeRoot TreeRoot; - -#ifndef _SQLITEINT_H_ -typedef unsigned char u8; -typedef unsigned short int u16; -typedef unsigned int u32; -typedef lsm_i64 i64; -typedef unsigned long long int u64; -#endif - -/* A page number is a 64-bit integer. */ -typedef i64 LsmPgno; - -#ifdef LSM_DEBUG -int lsmErrorBkpt(int); -#else -# define lsmErrorBkpt(x) (x) -#endif - -#define LSM_PROTOCOL_BKPT lsmErrorBkpt(LSM_PROTOCOL) -#define LSM_IOERR_BKPT lsmErrorBkpt(LSM_IOERR) -#define LSM_NOMEM_BKPT lsmErrorBkpt(LSM_NOMEM) -#define LSM_CORRUPT_BKPT lsmErrorBkpt(LSM_CORRUPT) -#define LSM_MISUSE_BKPT lsmErrorBkpt(LSM_MISUSE) - -#define unused_parameter(x) (void)(x) -#define array_size(x) (sizeof(x)/sizeof(x[0])) - - -/* The size of each shared-memory chunk */ -#define LSM_SHM_CHUNK_SIZE (32*1024) - -/* The number of bytes reserved at the start of each shm chunk for MM. */ -#define LSM_SHM_CHUNK_HDR (sizeof(ShmChunk)) - -/* The number of available read locks. */ -#define LSM_LOCK_NREADER 6 - -/* The number of available read-write client locks. */ -#define LSM_LOCK_NRWCLIENT 16 - -/* Lock definitions. -*/ -#define LSM_LOCK_DMS1 1 /* Serialize connect/disconnect ops */ -#define LSM_LOCK_DMS2 2 /* Read-write connections */ -#define LSM_LOCK_DMS3 3 /* Read-only connections */ -#define LSM_LOCK_WRITER 4 -#define LSM_LOCK_WORKER 5 -#define LSM_LOCK_CHECKPOINTER 6 -#define LSM_LOCK_ROTRANS 7 -#define LSM_LOCK_READER(i) ((i) + LSM_LOCK_ROTRANS + 1) -#define LSM_LOCK_RWCLIENT(i) ((i) + LSM_LOCK_READER(LSM_LOCK_NREADER)) - -#define LSM_N_LOCK LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT) - -/* -** Meta-page size and usable size. -*/ -#define LSM_META_PAGE_SIZE 4096 - -#define LSM_META_RW_PAGE_SIZE (LSM_META_PAGE_SIZE - LSM_N_LOCK) - -/* -** Hard limit on the number of free-list entries that may be stored in -** a checkpoint (the remainder are stored as a system record in the LSM). -** See also LSM_CONFIG_MAX_FREELIST. -*/ -#define LSM_MAX_FREELIST_ENTRIES 24 - -#define LSM_MAX_BLOCK_REDIRECTS 16 - -#define LSM_ATTEMPTS_BEFORE_PROTOCOL 10000 - - -/* -** Each entry stored in the LSM (or in-memory tree structure) has an -** associated mask of the following flags. -*/ -#define LSM_START_DELETE 0x01 /* Start of open-ended delete range */ -#define LSM_END_DELETE 0x02 /* End of open-ended delete range */ -#define LSM_POINT_DELETE 0x04 /* Delete this key */ -#define LSM_INSERT 0x08 /* Insert this key and value */ -#define LSM_SEPARATOR 0x10 /* True if entry is separator key only */ -#define LSM_SYSTEMKEY 0x20 /* True if entry is a system key (FREELIST) */ - -#define LSM_CONTIGUOUS 0x40 /* Used in lsm_tree.c */ - -/* -** A string that can grow by appending. -*/ -struct LsmString { - lsm_env *pEnv; /* Run-time environment */ - int n; /* Size of string. -1 indicates error */ - int nAlloc; /* Space allocated for z[] */ - char *z; /* The string content */ -}; - -typedef struct LsmFile LsmFile; -struct LsmFile { - lsm_file *pFile; - LsmFile *pNext; -}; - -/* -** An instance of the following type is used to store an ordered list of -** u32 values. -** -** Note: This is a place-holder implementation. It should be replaced by -** a version that avoids making a single large allocation when the array -** contains a large number of values. For this reason, the internals of -** this object should only manipulated by the intArrayXXX() functions in -** lsm_tree.c. -*/ -typedef struct IntArray IntArray; -struct IntArray { - int nAlloc; - int nArray; - u32 *aArray; -}; - -struct Redirect { - int n; /* Number of redirects */ - struct RedirectEntry { - int iFrom; - int iTo; - } *a; -}; - -/* -** An instance of this structure represents a point in the history of the -** tree structure to roll back to. Refer to comments in lsm_tree.c for -** details. -*/ -struct TreeMark { - u32 iRoot; /* Offset of root node in shm file */ - u32 nHeight; /* Current height of tree structure */ - u32 iWrite; /* Write offset in shm file */ - u32 nChunk; /* Number of chunks in shared-memory file */ - u32 iFirst; /* First chunk in linked list */ - u32 iNextShmid; /* Next id to allocate */ - int iRollback; /* Index in lsm->rollback to revert to */ -}; - -/* -** An instance of this structure represents a point in the database log. -*/ -struct LogMark { - i64 iOff; /* Offset into log (see lsm_log.c) */ - int nBuf; /* Size of in-memory buffer here */ - u8 aBuf[8]; /* Bytes of content in aBuf[] */ - u32 cksum0; /* Checksum 0 at offset (iOff-nBuf) */ - u32 cksum1; /* Checksum 1 at offset (iOff-nBuf) */ -}; - -struct TransMark { - TreeMark tree; - LogMark log; -}; - -/* -** A structure that defines the start and end offsets of a region in the -** log file. The size of the region in bytes is (iEnd - iStart), so if -** iEnd==iStart the region is zero bytes in size. -*/ -struct LogRegion { - i64 iStart; /* Start of region in log file */ - i64 iEnd; /* End of region in log file */ -}; - -struct DbLog { - u32 cksum0; /* Checksum 0 at offset iOff */ - u32 cksum1; /* Checksum 1 at offset iOff */ - i64 iSnapshotId; /* Log space has been reclaimed to this ss */ - LogRegion aRegion[3]; /* Log file regions (see docs in lsm_log.c) */ -}; - -struct TreeRoot { - u32 iRoot; - u32 nHeight; - u32 nByte; /* Total size of this tree in bytes */ - u32 iTransId; -}; - -/* -** Tree header structure. -*/ -struct TreeHeader { - u32 iUsedShmid; /* Id of first shm chunk used by this tree */ - u32 iNextShmid; /* Shm-id of next chunk allocated */ - u32 iFirst; /* Chunk number of smallest shm-id */ - u32 nChunk; /* Number of chunks in shared-memory file */ - TreeRoot root; /* Root and height of current tree */ - u32 iWrite; /* Write offset in shm file */ - TreeRoot oldroot; /* Root and height of the previous tree */ - u32 iOldShmid; /* Last shm-id used by previous tree */ - u32 iUsrVersion; /* get/set_user_version() value */ - i64 iOldLog; /* Log offset associated with old tree */ - u32 oldcksum0; - u32 oldcksum1; - DbLog log; /* Current layout of log file */ - u32 aCksum[2]; /* Checksums 1 and 2. */ -}; - -/* -** Database handle structure. -** -** mLock: -** A bitmask representing the locks currently held by the connection. -** An LSM database supports N distinct locks, where N is some number less -** than or equal to 32. Locks are numbered starting from 1 (see the -** definitions for LSM_LOCK_WRITER and co.). -** -** The least significant 32-bits in mLock represent EXCLUSIVE locks. The -** most significant are SHARED locks. So, if a connection holds a SHARED -** lock on lock region iLock, then the following is true: -** -** (mLock & ((iLock+32-1) << 1)) -** -** Or for an EXCLUSIVE lock: -** -** (mLock & ((iLock-1) << 1)) -** -** pCsr: -** Points to the head of a linked list that contains all currently open -** cursors. Once this list becomes empty, the user has no outstanding -** cursors and the database handle can be successfully closed. -** -** pCsrCache: -** This list contains cursor objects that have been closed using -** lsm_csr_close(). Each time a cursor is closed, it is shifted from -** the pCsr list to this list. When a new cursor is opened, this list -** is inspected to see if there exists a cursor object that can be -** reused. This is an optimization only. -*/ -struct lsm_db { - - /* Database handle configuration */ - lsm_env *pEnv; /* runtime environment */ - int (*xCmp)(void *, int, void *, int); /* Compare function */ - - /* Values configured by calls to lsm_config */ - int eSafety; /* LSM_SAFETY_OFF, NORMAL or FULL */ - int bAutowork; /* Configured by LSM_CONFIG_AUTOWORK */ - int nTreeLimit; /* Configured by LSM_CONFIG_AUTOFLUSH */ - int nMerge; /* Configured by LSM_CONFIG_AUTOMERGE */ - int bUseLog; /* Configured by LSM_CONFIG_USE_LOG */ - int nDfltPgsz; /* Configured by LSM_CONFIG_PAGE_SIZE */ - int nDfltBlksz; /* Configured by LSM_CONFIG_BLOCK_SIZE */ - int nMaxFreelist; /* Configured by LSM_CONFIG_MAX_FREELIST */ - int iMmap; /* Configured by LSM_CONFIG_MMAP */ - i64 nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ - int bMultiProc; /* Configured by L_C_MULTIPLE_PROCESSES */ - int bReadonly; /* Configured by LSM_CONFIG_READONLY */ - lsm_compress compress; /* Compression callbacks */ - lsm_compress_factory factory; /* Compression callback factory */ - - /* Sub-system handles */ - FileSystem *pFS; /* On-disk portion of database */ - Database *pDatabase; /* Database shared data */ - - int iRwclient; /* Read-write client lock held (-1 == none) */ - - /* Client transaction context */ - Snapshot *pClient; /* Client snapshot */ - int iReader; /* Read lock held (-1 == unlocked) */ - int bRoTrans; /* True if a read-only db trans is open */ - MultiCursor *pCsr; /* List of all open cursors */ - LogWriter *pLogWriter; /* Context for writing to the log file */ - int nTransOpen; /* Number of opened write transactions */ - int nTransAlloc; /* Allocated size of aTrans[] array */ - TransMark *aTrans; /* Array of marks for transaction rollback */ - IntArray rollback; /* List of tree-nodes to roll back */ - int bDiscardOld; /* True if lsmTreeDiscardOld() was called */ - - MultiCursor *pCsrCache; /* List of all closed cursors */ - - /* Worker context */ - Snapshot *pWorker; /* Worker snapshot (or NULL) */ - Freelist *pFreelist; /* See sortedNewToplevel() */ - int bUseFreelist; /* True to use pFreelist */ - int bIncrMerge; /* True if currently doing a merge */ - - int bInFactory; /* True if within factory.xFactory() */ - - /* Debugging message callback */ - void (*xLog)(void *, int, const char *); - void *pLogCtx; - - /* Work done notification callback */ - void (*xWork)(lsm_db *, void *); - void *pWorkCtx; - - u64 mLock; /* Mask of current locks. See lsmShmLock(). */ - lsm_db *pNext; /* Next connection to same database */ - - int nShm; /* Size of apShm[] array */ - void **apShm; /* Shared memory chunks */ - ShmHeader *pShmhdr; /* Live shared-memory header */ - TreeHeader treehdr; /* Local copy of tree-header */ - u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)]; -}; - -struct Segment { - LsmPgno iFirst; /* First page of this run */ - LsmPgno iLastPg; /* Last page of this run */ - LsmPgno iRoot; /* Root page number (if any) */ - LsmPgno nSize; /* Size of this run in pages */ - - Redirect *pRedirect; /* Block redirects (or NULL) */ -}; - -/* -** iSplitTopic/pSplitKey/nSplitKey: -** If nRight>0, this buffer contains a copy of the largest key that has -** already been written to the left-hand-side of the level. -*/ -struct Level { - Segment lhs; /* Left-hand (main) segment */ - int nRight; /* Size of apRight[] array */ - Segment *aRhs; /* Old segments being merged into this */ - int iSplitTopic; /* Split key topic (if nRight>0) */ - void *pSplitKey; /* Pointer to split-key (if nRight>0) */ - int nSplitKey; /* Number of bytes in split-key */ - - u16 iAge; /* Number of times data has been written */ - u16 flags; /* Mask of LEVEL_XXX bits */ - Merge *pMerge; /* Merge operation currently underway */ - Level *pNext; /* Next level in tree */ -}; - -/* -** The Level.flags field is set to a combination of the following bits. -** -** LEVEL_FREELIST_ONLY: -** Set if the level consists entirely of free-list entries. -** -** LEVEL_INCOMPLETE: -** This is set while a new toplevel level is being constructed. It is -** never set for any level other than a new toplevel. -*/ -#define LEVEL_FREELIST_ONLY 0x0001 -#define LEVEL_INCOMPLETE 0x0002 - - -/* -** A structure describing an ongoing merge. There is an instance of this -** structure for every Level currently undergoing a merge in the worker -** snapshot. -** -** It is assumed that code that uses an instance of this structure has -** access to the associated Level struct. -** -** iOutputOff: -** The byte offset to write to next within the last page of the -** output segment. -*/ -struct MergeInput { - LsmPgno iPg; /* Page on which next input is stored */ - int iCell; /* Cell containing next input to merge */ -}; -struct Merge { - int nInput; /* Number of input runs being merged */ - MergeInput *aInput; /* Array nInput entries in size */ - MergeInput splitkey; /* Location in file of current splitkey */ - int nSkip; /* Number of separators entries to skip */ - int iOutputOff; /* Write offset on output page */ - LsmPgno iCurrentPtr; /* Current pointer value */ -}; - -/* -** The first argument to this macro is a pointer to a Segment structure. -** Returns true if the structure instance indicates that the separators -** array is valid. -*/ -#define segmentHasSeparators(pSegment) ((pSegment)->sep.iFirst>0) - -/* -** The values that accompany the lock held by a database reader. -*/ -struct ShmReader { - u32 iTreeId; - i64 iLsmId; -}; - -/* -** An instance of this structure is stored in the first shared-memory -** page. The shared-memory header. -** -** bWriter: -** Immediately after opening a write transaction taking the WRITER lock, -** each writer client sets this flag. It is cleared right before the -** WRITER lock is relinquished. If a subsequent writer finds that this -** flag is already set when a write transaction is opened, this indicates -** that a previous writer failed mid-transaction. -** -** iMetaPage: -** If the database file does not contain a valid, synced, checkpoint, this -** value is set to 0. Otherwise, it is set to the meta-page number that -** contains the most recently written checkpoint (either 1 or 2). -** -** hdr1, hdr2: -** The two copies of the in-memory tree header. Two copies are required -** in case a writer fails while updating one of them. -*/ -struct ShmHeader { - u32 aSnap1[LSM_META_PAGE_SIZE / 4]; - u32 aSnap2[LSM_META_PAGE_SIZE / 4]; - u32 bWriter; - u32 iMetaPage; - TreeHeader hdr1; - TreeHeader hdr2; - ShmReader aReader[LSM_LOCK_NREADER]; -}; - -/* -** An instance of this structure is stored at the start of each shared-memory -** chunk except the first (which is the header chunk - see above). -*/ -struct ShmChunk { - u32 iShmid; - u32 iNext; -}; - -/* -** Maximum number of shared-memory chunks allowed in the *-shm file. Since -** each shared-memory chunk is 32KB in size, this is a theoretical limit only. -*/ -#define LSM_MAX_SHMCHUNKS (1<<30) - -/* Return true if shm-sequence "a" is larger than or equal to "b" */ -#define shm_sequence_ge(a, b) (((u32)a-(u32)b) < LSM_MAX_SHMCHUNKS) - -#define LSM_APPLIST_SZ 4 - -/* -** An instance of the following structure stores the in-memory part of -** the current free block list. This structure is to the free block list -** as the in-memory tree is to the users database content. The contents -** of the free block list is found by merging the in-memory components -** with those stored in the LSM, just as the contents of the database is -** found by merging the in-memory tree with the user data entries in the -** LSM. -** -** Each FreelistEntry structure in the array represents either an insert -** or delete operation on the free-list. For deletes, the FreelistEntry.iId -** field is set to -1. For inserts, it is set to zero or greater. -** -** The array of FreelistEntry structures is always sorted in order of -** block number (ascending). -** -** When the in-memory free block list is written into the LSM, each insert -** operation is written separately. The entry key is the bitwise inverse -** of the block number as a 32-bit big-endian integer. This is done so that -** the entries in the LSM are sorted in descending order of block id. -** The associated value is the snapshot id, formated as a varint. -*/ -struct Freelist { - FreelistEntry *aEntry; /* Free list entries */ - int nEntry; /* Number of valid slots in aEntry[] */ - int nAlloc; /* Allocated size of aEntry[] */ -}; -struct FreelistEntry { - u32 iBlk; /* Block number */ - i64 iId; /* Largest snapshot id to use this block */ -}; - -/* -** A snapshot of a database. A snapshot contains all the information required -** to read or write a database file on disk. See the description of struct -** Database below for further details. -*/ -struct Snapshot { - Database *pDatabase; /* Database this snapshot belongs to */ - u32 iCmpId; /* Id of compression scheme */ - Level *pLevel; /* Pointer to level 0 of snapshot (or NULL) */ - i64 iId; /* Snapshot id */ - i64 iLogOff; /* Log file offset */ - Redirect redirect; /* Block redirection array */ - - /* Used by worker snapshots only */ - int nBlock; /* Number of blocks in database file */ - LsmPgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */ - Freelist freelist; /* Free block list */ - u32 nWrite; /* Total number of pages written to disk */ -}; -#define LSM_INITIAL_SNAPSHOT_ID 11 - -/* -** Functions from file "lsm_ckpt.c". -*/ -int lsmCheckpointWrite(lsm_db *, u32 *); -int lsmCheckpointLevels(lsm_db *, int, void **, int *); -int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal); - -int lsmCheckpointRecover(lsm_db *); -int lsmCheckpointDeserialize(lsm_db *, int, u32 *, Snapshot **); - -int lsmCheckpointLoadWorker(lsm_db *pDb); -int lsmCheckpointStore(lsm_db *pDb, int); - -int lsmCheckpointLoad(lsm_db *pDb, int *); -int lsmCheckpointLoadOk(lsm_db *pDb, int); -int lsmCheckpointClientCacheOk(lsm_db *); - -u32 lsmCheckpointNBlock(u32 *); -i64 lsmCheckpointId(u32 *, int); -u32 lsmCheckpointNWrite(u32 *, int); -i64 lsmCheckpointLogOffset(u32 *); -int lsmCheckpointPgsz(u32 *); -int lsmCheckpointBlksz(u32 *); -void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); -void lsmCheckpointZeroLogoffset(lsm_db *); - -int lsmCheckpointSaveWorker(lsm_db *pDb, int); -int lsmDatabaseFull(lsm_db *pDb); -int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite); - -int lsmCheckpointSize(lsm_db *db, int *pnByte); - -int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId); - -/* -** Functions from file "lsm_tree.c". -*/ -int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree); -void lsmTreeRelease(lsm_env *, Tree *); -int lsmTreeInit(lsm_db *); -int lsmTreeRepair(lsm_db *); - -void lsmTreeMakeOld(lsm_db *pDb); -void lsmTreeDiscardOld(lsm_db *pDb); -int lsmTreeHasOld(lsm_db *pDb); - -int lsmTreeSize(lsm_db *); -int lsmTreeEndTransaction(lsm_db *pDb, int bCommit); -int lsmTreeLoadHeader(lsm_db *pDb, int *); -int lsmTreeLoadHeaderOk(lsm_db *, int); - -int lsmTreeInsert(lsm_db *pDb, void *pKey, int nKey, void *pVal, int nVal); -int lsmTreeDelete(lsm_db *db, void *pKey1, int nKey1, void *pKey2, int nKey2); -void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark); -void lsmTreeMark(lsm_db *pDb, TreeMark *pMark); - -int lsmTreeCursorNew(lsm_db *pDb, int, TreeCursor **); -void lsmTreeCursorDestroy(TreeCursor *); - -int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes); -int lsmTreeCursorNext(TreeCursor *pCsr); -int lsmTreeCursorPrev(TreeCursor *pCsr); -int lsmTreeCursorEnd(TreeCursor *pCsr, int bLast); -void lsmTreeCursorReset(TreeCursor *pCsr); -int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey); -int lsmTreeCursorFlags(TreeCursor *pCsr); -int lsmTreeCursorValue(TreeCursor *pCsr, void **ppVal, int *pnVal); -int lsmTreeCursorValid(TreeCursor *pCsr); -int lsmTreeCursorSave(TreeCursor *pCsr); - -void lsmFlagsToString(int flags, char *zFlags); - -/* -** Functions from file "mem.c". -*/ -void *lsmMalloc(lsm_env*, size_t); -void lsmFree(lsm_env*, void *); -void *lsmRealloc(lsm_env*, void *, size_t); -void *lsmReallocOrFree(lsm_env*, void *, size_t); -void *lsmReallocOrFreeRc(lsm_env *, void *, size_t, int *); - -void *lsmMallocZeroRc(lsm_env*, size_t, int *); -void *lsmMallocRc(lsm_env*, size_t, int *); - -void *lsmMallocZero(lsm_env *pEnv, size_t); -char *lsmMallocStrdup(lsm_env *pEnv, const char *); - -/* -** Functions from file "lsm_mutex.c". -*/ -int lsmMutexStatic(lsm_env*, int, lsm_mutex **); -int lsmMutexNew(lsm_env*, lsm_mutex **); -void lsmMutexDel(lsm_env*, lsm_mutex *); -void lsmMutexEnter(lsm_env*, lsm_mutex *); -int lsmMutexTry(lsm_env*, lsm_mutex *); -void lsmMutexLeave(lsm_env*, lsm_mutex *); - -#ifndef NDEBUG -int lsmMutexHeld(lsm_env *, lsm_mutex *); -int lsmMutexNotHeld(lsm_env *, lsm_mutex *); -#endif - -/************************************************************************** -** Start of functions from "lsm_file.c". -*/ -int lsmFsOpen(lsm_db *, const char *, int); -int lsmFsOpenLog(lsm_db *, int *); -void lsmFsCloseLog(lsm_db *); -void lsmFsClose(FileSystem *); - -int lsmFsUnmap(FileSystem *); - -int lsmFsConfigure(lsm_db *db); - -int lsmFsBlockSize(FileSystem *); -void lsmFsSetBlockSize(FileSystem *, int); -int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom); - -int lsmFsPageSize(FileSystem *); -void lsmFsSetPageSize(FileSystem *, int); - -int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId); - -/* Creating, populating, gobbling and deleting sorted runs. */ -void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int); -int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); -int lsmFsSortedFinish(FileSystem *, Segment *); -int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **); -int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *); - -/* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */ -lsm_env *lsmFsEnv(FileSystem *); -lsm_env *lsmPageEnv(Page *); -FileSystem *lsmPageFS(Page *); - -int lsmFsSectorSize(FileSystem *); - -void lsmSortedSplitkey(lsm_db *, Level *, int *); - -/* Reading sorted run content. */ -int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg); -int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **); -int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **); - -u8 *lsmFsPageData(Page *, int *); -int lsmFsPageRelease(Page *); -int lsmFsPagePersist(Page *); -void lsmFsPageRef(Page *); -LsmPgno lsmFsPageNumber(Page *); - -int lsmFsNRead(FileSystem *); -int lsmFsNWrite(FileSystem *); - -int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **); -int lsmFsMetaPageRelease(MetaPage *); -u8 *lsmFsMetaPageData(MetaPage *, int *); - -#ifdef LSM_DEBUG -int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg); -int lsmFsIntegrityCheck(lsm_db *); -#endif - -LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno); - -int lsmFsPageWritable(Page *); - -/* Functions to read, write and sync the log file. */ -int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr); -int lsmFsSyncLog(FileSystem *pFS); -int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr); -int lsmFsTruncateLog(FileSystem *pFS, i64 nByte); -int lsmFsTruncateDb(FileSystem *pFS, i64 nByte); -int lsmFsCloseAndDeleteLog(FileSystem *pFS); - -LsmFile *lsmFsDeferClose(FileSystem *pFS); - -/* And to sync the db file */ -int lsmFsSyncDb(FileSystem *, int); - -void lsmFsFlushWaiting(FileSystem *, int *); - -/* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ -int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz); -int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut); -int lsmConfigMmap(lsm_db *pDb, int *piParam); - -int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **); -int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile); -int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock); -int lsmEnvTestLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int nLock, int); - -int lsmEnvShmMap(lsm_env *, lsm_file *, int, int, void **); -void lsmEnvShmBarrier(lsm_env *); -void lsmEnvShmUnmap(lsm_env *, lsm_file *, int); - -void lsmEnvSleep(lsm_env *, int); - -int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal); - -int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *); - -void lsmFsPurgeCache(FileSystem *); - -/* -** End of functions from "lsm_file.c". -**************************************************************************/ - -/* -** Functions from file "lsm_sorted.c". -*/ -int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **); -void lsmSortedCleanup(lsm_db *); -int lsmSortedAutoWork(lsm_db *, int nUnit); - -int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); - -int lsmSaveWorker(lsm_db *, int); - -int lsmFlushTreeToDisk(lsm_db *pDb); - -void lsmSortedRemap(lsm_db *pDb); - -void lsmSortedFreeLevel(lsm_env *pEnv, Level *); - -int lsmSortedAdvanceAll(lsm_db *pDb); - -int lsmSortedLoadMerge(lsm_db *, Level *, u32 *, int *); -int lsmSortedLoadFreelist(lsm_db *pDb, void **, int *); - -void *lsmSortedSplitKey(Level *pLevel, int *pnByte); - -void lsmSortedSaveTreeCursors(lsm_db *); - -int lsmMCursorNew(lsm_db *, MultiCursor **); -void lsmMCursorClose(MultiCursor *, int); -int lsmMCursorSeek(MultiCursor *, int, void *, int , int); -int lsmMCursorFirst(MultiCursor *); -int lsmMCursorPrev(MultiCursor *); -int lsmMCursorLast(MultiCursor *); -int lsmMCursorValid(MultiCursor *); -int lsmMCursorNext(MultiCursor *); -int lsmMCursorKey(MultiCursor *, void **, int *); -int lsmMCursorValue(MultiCursor *, void **, int *); -int lsmMCursorType(MultiCursor *, int *); -lsm_db *lsmMCursorDb(MultiCursor *); -void lsmMCursorFreeCache(lsm_db *); - -int lsmSaveCursors(lsm_db *pDb); -int lsmRestoreCursors(lsm_db *pDb); - -void lsmSortedDumpStructure(lsm_db *pDb, Snapshot *, int, int, const char *); -void lsmFsDumpBlocklists(lsm_db *); - -void lsmSortedExpandBtreePage(Page *pPg, int nOrig); - -void lsmPutU32(u8 *, u32); -u32 lsmGetU32(u8 *); -u64 lsmGetU64(u8 *); - -/* -** Functions from "lsm_varint.c". -*/ -int lsmVarintPut32(u8 *, int); -int lsmVarintGet32(u8 *, int *); -int lsmVarintPut64(u8 *aData, i64 iVal); -int lsmVarintGet64(const u8 *aData, i64 *piVal); - -int lsmVarintLen64(i64); - -int lsmVarintLen32(int); -int lsmVarintSize(u8 c); - -/* -** Functions from file "main.c". -*/ -void lsmLogMessage(lsm_db *, int, const char *, ...); -int lsmInfoFreelist(lsm_db *pDb, char **pzOut); - -/* -** Functions from file "lsm_log.c". -*/ -int lsmLogBegin(lsm_db *pDb); -int lsmLogWrite(lsm_db *, int, void *, int, void *, int); -int lsmLogCommit(lsm_db *); -void lsmLogEnd(lsm_db *pDb, int bCommit); -void lsmLogTell(lsm_db *, LogMark *); -void lsmLogSeek(lsm_db *, LogMark *); -void lsmLogClose(lsm_db *); - -int lsmLogRecover(lsm_db *); -int lsmInfoLogStructure(lsm_db *pDb, char **pzVal); - -/* Valid values for the second argument to lsmLogWrite(). */ -#define LSM_WRITE 0x06 -#define LSM_DELETE 0x08 -#define LSM_DRANGE 0x0A - -/************************************************************************** -** Functions from file "lsm_shared.c". -*/ - -int lsmDbDatabaseConnect(lsm_db*, const char *); -void lsmDbDatabaseRelease(lsm_db *); - -int lsmBeginReadTrans(lsm_db *); -int lsmBeginWriteTrans(lsm_db *); -int lsmBeginFlush(lsm_db *); - -int lsmDetectRoTrans(lsm_db *db, int *); -int lsmBeginRoTrans(lsm_db *db); - -int lsmBeginWork(lsm_db *); -void lsmFinishWork(lsm_db *, int, int *); - -int lsmFinishRecovery(lsm_db *); -void lsmFinishReadTrans(lsm_db *); -int lsmFinishWriteTrans(lsm_db *, int); -int lsmFinishFlush(lsm_db *, int); - -int lsmSnapshotSetFreelist(lsm_db *, int *, int); - -Snapshot *lsmDbSnapshotClient(lsm_db *); -Snapshot *lsmDbSnapshotWorker(lsm_db *); - -void lsmSnapshotSetCkptid(Snapshot *, i64); - -Level *lsmDbSnapshotLevel(Snapshot *); -void lsmDbSnapshotSetLevel(Snapshot *, Level *); - -void lsmDbRecoveryComplete(lsm_db *, int); - -int lsmBlockAllocate(lsm_db *, int, int *); -int lsmBlockFree(lsm_db *, int); -int lsmBlockRefree(lsm_db *, int); - -void lsmFreelistDeltaBegin(lsm_db *); -void lsmFreelistDeltaEnd(lsm_db *); -int lsmFreelistDelta(lsm_db *pDb); - -DbLog *lsmDatabaseLog(lsm_db *pDb); - -#ifdef LSM_DEBUG - int lsmHoldingClientMutex(lsm_db *pDb); - int lsmShmAssertLock(lsm_db *db, int iLock, int eOp); - int lsmShmAssertWorker(lsm_db *db); -#endif - -void lsmFreeSnapshot(lsm_env *, Snapshot *); - - -/* Candidate values for the 3rd argument to lsmShmLock() */ -#define LSM_LOCK_UNLOCK 0 -#define LSM_LOCK_SHARED 1 -#define LSM_LOCK_EXCL 2 - -int lsmShmCacheChunks(lsm_db *db, int nChunk); -int lsmShmLock(lsm_db *db, int iLock, int eOp, int bBlock); -int lsmShmTestLock(lsm_db *db, int iLock, int nLock, int eOp); -void lsmShmBarrier(lsm_db *db); - -#ifdef LSM_DEBUG -void lsmShmHasLock(lsm_db *db, int iLock, int eOp); -#else -# define lsmShmHasLock(x,y,z) -#endif - -int lsmReadlock(lsm_db *, i64 iLsm, u32 iShmMin, u32 iShmMax); - -int lsmLsmInUse(lsm_db *db, i64 iLsmId, int *pbInUse); -int lsmTreeInUse(lsm_db *db, u32 iLsmId, int *pbInUse); -int lsmFreelistAppend(lsm_env *pEnv, Freelist *p, int iBlk, i64 iId); - -int lsmDbMultiProc(lsm_db *); -void lsmDbDeferredClose(lsm_db *, lsm_file *, LsmFile *); -LsmFile *lsmDbRecycleFd(lsm_db *); - -int lsmWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); - -int lsmCheckCompressionId(lsm_db *, u32); - - -/************************************************************************** -** functions in lsm_str.c -*/ -void lsmStringInit(LsmString*, lsm_env *pEnv); -int lsmStringExtend(LsmString*, int); -int lsmStringAppend(LsmString*, const char *, int); -void lsmStringVAppendf(LsmString*, const char *zFormat, va_list, va_list); -void lsmStringAppendf(LsmString*, const char *zFormat, ...); -void lsmStringClear(LsmString*); -char *lsmMallocPrintf(lsm_env*, const char*, ...); -int lsmStringBinAppend(LsmString *pStr, const u8 *a, int n); - -int lsmStrlen(const char *zName); - - - -/* -** Round up a number to the next larger multiple of 8. This is used -** to force 8-byte alignment on 64-bit architectures. -*/ -#define ROUND8(x) (((x)+7)&~7) - -#define LSM_MIN(x,y) ((x)>(y) ? (y) : (x)) -#define LSM_MAX(x,y) ((x)>(y) ? (x) : (y)) - -#endif diff --git a/ext/lsm1/lsm_ckpt.c b/ext/lsm1/lsm_ckpt.c deleted file mode 100644 index dbfa1a61f..000000000 --- a/ext/lsm1/lsm_ckpt.c +++ /dev/null @@ -1,1239 +0,0 @@ -/* -** 2011-09-11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains code to read and write checkpoints. -** -** A checkpoint represents the database layout at a single point in time. -** It includes a log offset. When an existing database is opened, the -** current state is determined by reading the newest checkpoint and updating -** it with all committed transactions from the log that follow the specified -** offset. -*/ -#include "lsmInt.h" - -/* -** CHECKPOINT BLOB FORMAT: -** -** A checkpoint blob is a series of unsigned 32-bit integers stored in -** big-endian byte order. As follows: -** -** Checkpoint header (see the CKPT_HDR_XXX #defines): -** -** 1. The checkpoint id MSW. -** 2. The checkpoint id LSW. -** 3. The number of integer values in the entire checkpoint, including -** the two checksum values. -** 4. The compression scheme id. -** 5. The total number of blocks in the database. -** 6. The block size. -** 7. The number of levels. -** 8. The nominal database page size. -** 9. The number of pages (in total) written to the database file. -** -** Log pointer: -** -** 1. The log offset MSW. -** 2. The log offset LSW. -** 3. Log checksum 0. -** 4. Log checksum 1. -** -** Note that the "log offset" is not the literal byte offset. Instead, -** it is the byte offset multiplied by 2, with least significant bit -** toggled each time the log pointer value is changed. This is to make -** sure that this field changes each time the log pointer is updated, -** even if the log file itself is disabled. See lsmTreeMakeOld(). -** -** See ckptExportLog() and ckptImportLog(). -** -** Append points: -** -** 8 integers (4 * 64-bit page numbers). See ckptExportAppendlist(). -** -** For each level in the database, a level record. Formatted as follows: -** -** 0. Age of the level (least significant 16-bits). And flags mask (most -** significant 16-bits). -** 1. The number of right-hand segments (nRight, possibly 0), -** 2. Segment record for left-hand segment (8 integers defined below), -** 3. Segment record for each right-hand segment (8 integers defined below), -** 4. If nRight>0, The number of segments involved in the merge -** 5. if nRight>0, Current nSkip value (see Merge structure defn.), -** 6. For each segment in the merge: -** 5a. Page number of next cell to read during merge (this field -** is 64-bits - 2 integers) -** 5b. Cell number of next cell to read during merge -** 7. Page containing current split-key (64-bits - 2 integers). -** 8. Cell within page containing current split-key. -** 9. Current pointer value (64-bits - 2 integers). -** -** The block redirect array: -** -** 1. Number of redirections (maximum LSM_MAX_BLOCK_REDIRECTS). -** 2. For each redirection: -** a. "from" block number -** b. "to" block number -** -** The in-memory freelist entries. Each entry is either an insert or a -** delete. The in-memory freelist is to the free-block-list as the -** in-memory tree is to the users database content. -** -** 1. Number of free-list entries stored in checkpoint header. -** 2. Number of free blocks (in total). -** 3. Total number of blocks freed during database lifetime. -** 4. For each entry: -** 2a. Block number of free block. -** 2b. A 64-bit integer (MSW followed by LSW). -1 for a delete entry, -** or the associated checkpoint id for an insert. -** -** The checksum: -** -** 1. Checksum value 1. -** 2. Checksum value 2. -** -** In the above, a segment record consists of the following four 64-bit -** fields (converted to 2 * u32 by storing the MSW followed by LSW): -** -** 1. First page of array, -** 2. Last page of array, -** 3. Root page of array (or 0), -** 4. Size of array in pages. -*/ - -/* -** LARGE NUMBERS OF LEVEL RECORDS: -** -** A limit on the number of rhs segments that may be present in the database -** file. Defining this limit ensures that all level records fit within -** the 4096 byte limit for checkpoint blobs. -** -** The number of right-hand-side segments in a database is counted as -** follows: -** -** * For each level in the database not undergoing a merge, add 1. -** -** * For each level in the database that is undergoing a merge, add -** the number of segments on the rhs of the level. -** -** A level record not undergoing a merge is 10 integers. A level record -** with nRhs rhs segments and (nRhs+1) input segments (i.e. including the -** separators from the next level) is (11*nRhs+20) integers. The maximum -** per right-hand-side level is therefore 21 integers. So the maximum -** size of all level records in a checkpoint is 21*40=820 integers. -** -** TODO: Before pointer values were changed from 32 to 64 bits, the above -** used to come to 420 bytes - leaving significant space for a free-list -** prefix. No more. To fix this, reduce the size of the level records in -** a db snapshot, and improve management of the free-list tail in -** lsm_sorted.c. -*/ -#define LSM_MAX_RHS_SEGMENTS 40 - -/* -** LARGE NUMBERS OF FREELIST ENTRIES: -** -** There is also a limit (LSM_MAX_FREELIST_ENTRIES - defined in lsmInt.h) -** on the number of free-list entries stored in a checkpoint. Since each -** free-list entry consists of 3 integers, the maximum free-list size is -** 3*100=300 integers. Combined with the limit on rhs segments defined -** above, this ensures that a checkpoint always fits within a 4096 byte -** meta page. -** -** If the database contains more than 100 free blocks, the "overflow" flag -** in the checkpoint header is set and the remainder are stored in the -** system FREELIST entry in the LSM (along with user data). The value -** accompanying the FREELIST key in the LSM is, like a checkpoint, an array -** of 32-bit big-endian integers. As follows: -** -** For each entry: -** a. Block number of free block. -** b. MSW of associated checkpoint id. -** c. LSW of associated checkpoint id. -** -** The number of entries is not required - it is implied by the size of the -** value blob containing the integer array. -** -** Note that the limit defined by LSM_MAX_FREELIST_ENTRIES is a hard limit. -** The actual value used may be configured using LSM_CONFIG_MAX_FREELIST. -*/ - -/* -** The argument to this macro must be of type u32. On a little-endian -** architecture, it returns the u32 value that results from interpreting -** the 4 bytes as a big-endian value. On a big-endian architecture, it -** returns the value that would be produced by interpreting the 4 bytes -** of the input value as a little-endian integer. -*/ -#define BYTESWAP32(x) ( \ - (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ - + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ -) - -static const int one = 1; -#define LSM_LITTLE_ENDIAN (*(u8 *)(&one)) - -/* Sizes, in integers, of various parts of the checkpoint. */ -#define CKPT_HDR_SIZE 9 -#define CKPT_LOGPTR_SIZE 4 -#define CKPT_APPENDLIST_SIZE (LSM_APPLIST_SZ * 2) - -/* A #define to describe each integer in the checkpoint header. */ -#define CKPT_HDR_ID_MSW 0 -#define CKPT_HDR_ID_LSW 1 -#define CKPT_HDR_NCKPT 2 -#define CKPT_HDR_CMPID 3 -#define CKPT_HDR_NBLOCK 4 -#define CKPT_HDR_BLKSZ 5 -#define CKPT_HDR_NLEVEL 6 -#define CKPT_HDR_PGSZ 7 -#define CKPT_HDR_NWRITE 8 - -#define CKPT_HDR_LO_MSW 9 -#define CKPT_HDR_LO_LSW 10 -#define CKPT_HDR_LO_CKSUM1 11 -#define CKPT_HDR_LO_CKSUM2 12 - -typedef struct CkptBuffer CkptBuffer; - -/* -** Dynamic buffer used to accumulate data for a checkpoint. -*/ -struct CkptBuffer { - lsm_env *pEnv; - int nAlloc; - u32 *aCkpt; -}; - -/* -** Calculate the checksum of the checkpoint specified by arguments aCkpt and -** nCkpt. Store the checksum in *piCksum1 and *piCksum2 before returning. -** -** The value of the nCkpt parameter includes the two checksum values at -** the end of the checkpoint. They are not used as inputs to the checksum -** calculation. The checksum is based on the array of (nCkpt-2) integers -** at aCkpt[]. -*/ -static void ckptChecksum(u32 *aCkpt, u32 nCkpt, u32 *piCksum1, u32 *piCksum2){ - u32 i; - u32 cksum1 = 1; - u32 cksum2 = 2; - - if( nCkpt % 2 ){ - cksum1 += aCkpt[nCkpt-3] & 0x0000FFFF; - cksum2 += aCkpt[nCkpt-3] & 0xFFFF0000; - } - - for(i=0; (i+3)<nCkpt; i+=2){ - cksum1 += cksum2 + aCkpt[i]; - cksum2 += cksum1 + aCkpt[i+1]; - } - - *piCksum1 = cksum1; - *piCksum2 = cksum2; -} - -/* -** Set integer iIdx of the checkpoint accumulating in buffer *p to iVal. -*/ -static void ckptSetValue(CkptBuffer *p, int iIdx, u32 iVal, int *pRc){ - if( *pRc ) return; - if( iIdx>=p->nAlloc ){ - int nNew = LSM_MAX(8, iIdx*2); - p->aCkpt = (u32 *)lsmReallocOrFree(p->pEnv, p->aCkpt, nNew*sizeof(u32)); - if( !p->aCkpt ){ - *pRc = LSM_NOMEM_BKPT; - return; - } - p->nAlloc = nNew; - } - p->aCkpt[iIdx] = iVal; -} - -/* -** Argument aInt points to an array nInt elements in size. Switch the -** endian-ness of each element of the array. -*/ -static void ckptChangeEndianness(u32 *aInt, int nInt){ - if( LSM_LITTLE_ENDIAN ){ - int i; - for(i=0; i<nInt; i++) aInt[i] = BYTESWAP32(aInt[i]); - } -} - -/* -** Object *p contains a checkpoint in native byte-order. The checkpoint is -** nCkpt integers in size, not including any checksum. This function sets -** the two checksum elements of the checkpoint accordingly. -*/ -static void ckptAddChecksum(CkptBuffer *p, int nCkpt, int *pRc){ - if( *pRc==LSM_OK ){ - u32 aCksum[2] = {0, 0}; - ckptChecksum(p->aCkpt, nCkpt+2, &aCksum[0], &aCksum[1]); - ckptSetValue(p, nCkpt, aCksum[0], pRc); - ckptSetValue(p, nCkpt+1, aCksum[1], pRc); - } -} - -static void ckptAppend64(CkptBuffer *p, int *piOut, i64 iVal, int *pRc){ - int iOut = *piOut; - ckptSetValue(p, iOut++, (iVal >> 32) & 0xFFFFFFFF, pRc); - ckptSetValue(p, iOut++, (iVal & 0xFFFFFFFF), pRc); - *piOut = iOut; -} - -static i64 ckptRead64(u32 *a){ - return (((i64)a[0]) << 32) + (i64)a[1]; -} - -static i64 ckptGobble64(u32 *a, int *piIn){ - int iIn = *piIn; - *piIn += 2; - return ckptRead64(&a[iIn]); -} - - -/* -** Append a 6-value segment record corresponding to pSeg to the checkpoint -** buffer passed as the third argument. -*/ -static void ckptExportSegment( - Segment *pSeg, - CkptBuffer *p, - int *piOut, - int *pRc -){ - ckptAppend64(p, piOut, pSeg->iFirst, pRc); - ckptAppend64(p, piOut, pSeg->iLastPg, pRc); - ckptAppend64(p, piOut, pSeg->iRoot, pRc); - ckptAppend64(p, piOut, pSeg->nSize, pRc); -} - -static void ckptExportLevel( - Level *pLevel, /* Level object to serialize */ - CkptBuffer *p, /* Append new level record to this ckpt */ - int *piOut, /* IN/OUT: Size of checkpoint so far */ - int *pRc /* IN/OUT: Error code */ -){ - int iOut = *piOut; - Merge *pMerge; - - pMerge = pLevel->pMerge; - ckptSetValue(p, iOut++, (u32)pLevel->iAge + (u32)(pLevel->flags<<16), pRc); - ckptSetValue(p, iOut++, pLevel->nRight, pRc); - ckptExportSegment(&pLevel->lhs, p, &iOut, pRc); - - assert( (pLevel->nRight>0)==(pMerge!=0) ); - if( pMerge ){ - int i; - for(i=0; i<pLevel->nRight; i++){ - ckptExportSegment(&pLevel->aRhs[i], p, &iOut, pRc); - } - assert( pMerge->nInput==pLevel->nRight - || pMerge->nInput==pLevel->nRight+1 - ); - ckptSetValue(p, iOut++, pMerge->nInput, pRc); - ckptSetValue(p, iOut++, pMerge->nSkip, pRc); - for(i=0; i<pMerge->nInput; i++){ - ckptAppend64(p, &iOut, pMerge->aInput[i].iPg, pRc); - ckptSetValue(p, iOut++, pMerge->aInput[i].iCell, pRc); - } - ckptAppend64(p, &iOut, pMerge->splitkey.iPg, pRc); - ckptSetValue(p, iOut++, pMerge->splitkey.iCell, pRc); - ckptAppend64(p, &iOut, pMerge->iCurrentPtr, pRc); - } - - *piOut = iOut; -} - -/* -** Populate the log offset fields of the checkpoint buffer. 4 values. -*/ -static void ckptExportLog( - lsm_db *pDb, - int bFlush, - CkptBuffer *p, - int *piOut, - int *pRc -){ - int iOut = *piOut; - - assert( iOut==CKPT_HDR_LO_MSW ); - - if( bFlush ){ - i64 iOff = pDb->treehdr.iOldLog; - ckptAppend64(p, &iOut, iOff, pRc); - ckptSetValue(p, iOut++, pDb->treehdr.oldcksum0, pRc); - ckptSetValue(p, iOut++, pDb->treehdr.oldcksum1, pRc); - }else{ - for(; iOut<=CKPT_HDR_LO_CKSUM2; iOut++){ - ckptSetValue(p, iOut, pDb->pShmhdr->aSnap2[iOut], pRc); - } - } - - assert( *pRc || iOut==CKPT_HDR_LO_CKSUM2+1 ); - *piOut = iOut; -} - -static void ckptExportAppendlist( - lsm_db *db, /* Database connection */ - CkptBuffer *p, /* Checkpoint buffer to write to */ - int *piOut, /* IN/OUT: Offset within checkpoint buffer */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - LsmPgno *aiAppend = db->pWorker->aiAppend; - - for(i=0; i<LSM_APPLIST_SZ; i++){ - ckptAppend64(p, piOut, aiAppend[i], pRc); - } -}; - -static int ckptExportSnapshot( - lsm_db *pDb, /* Connection handle */ - int bLog, /* True to update log-offset fields */ - i64 iId, /* Checkpoint id */ - int bCksum, /* If true, include checksums */ - void **ppCkpt, /* OUT: Buffer containing checkpoint */ - int *pnCkpt /* OUT: Size of checkpoint in bytes */ -){ - int rc = LSM_OK; /* Return Code */ - FileSystem *pFS = pDb->pFS; /* File system object */ - Snapshot *pSnap = pDb->pWorker; /* Worker snapshot */ - int nLevel = 0; /* Number of levels in checkpoint */ - int iLevel; /* Used to count out nLevel levels */ - int iOut = 0; /* Current offset in aCkpt[] */ - Level *pLevel; /* Level iterator */ - int i; /* Iterator used while serializing freelist */ - CkptBuffer ckpt; - - /* Initialize the output buffer */ - memset(&ckpt, 0, sizeof(CkptBuffer)); - ckpt.pEnv = pDb->pEnv; - iOut = CKPT_HDR_SIZE; - - /* Write the log offset into the checkpoint. */ - ckptExportLog(pDb, bLog, &ckpt, &iOut, &rc); - - /* Write the append-point list */ - ckptExportAppendlist(pDb, &ckpt, &iOut, &rc); - - /* Figure out how many levels will be written to the checkpoint. */ - for(pLevel=lsmDbSnapshotLevel(pSnap); pLevel; pLevel=pLevel->pNext) nLevel++; - - /* Serialize nLevel levels. */ - iLevel = 0; - for(pLevel=lsmDbSnapshotLevel(pSnap); iLevel<nLevel; pLevel=pLevel->pNext){ - ckptExportLevel(pLevel, &ckpt, &iOut, &rc); - iLevel++; - } - - /* Write the block-redirect list */ - ckptSetValue(&ckpt, iOut++, pSnap->redirect.n, &rc); - for(i=0; i<pSnap->redirect.n; i++){ - ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iFrom, &rc); - ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iTo, &rc); - } - - /* Write the freelist */ - assert( pSnap->freelist.nEntry<=pDb->nMaxFreelist ); - if( rc==LSM_OK ){ - int nFree = pSnap->freelist.nEntry; - ckptSetValue(&ckpt, iOut++, nFree, &rc); - for(i=0; i<nFree; i++){ - FreelistEntry *p = &pSnap->freelist.aEntry[i]; - ckptSetValue(&ckpt, iOut++, p->iBlk, &rc); - ckptSetValue(&ckpt, iOut++, (p->iId >> 32) & 0xFFFFFFFF, &rc); - ckptSetValue(&ckpt, iOut++, p->iId & 0xFFFFFFFF, &rc); - } - } - - /* Write the checkpoint header */ - assert( iId>=0 ); - assert( pSnap->iCmpId==pDb->compress.iId - || pSnap->iCmpId==LSM_COMPRESSION_EMPTY - ); - ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); - ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); - ckptSetValue(&ckpt, CKPT_HDR_CMPID, pDb->compress.iId, &rc); - ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); - ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); - ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); - - if( bCksum ){ - ckptAddChecksum(&ckpt, iOut, &rc); - }else{ - ckptSetValue(&ckpt, iOut, 0, &rc); - ckptSetValue(&ckpt, iOut+1, 0, &rc); - } - iOut += 2; - assert( iOut<=1024 ); - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, rc, - "ckptExportSnapshot(): id=%lld freelist: %d", iId, pSnap->freelist.nEntry - ); - for(i=0; i<pSnap->freelist.nEntry; i++){ - lsmLogMessage(pDb, rc, - "ckptExportSnapshot(): iBlk=%d id=%lld", - pSnap->freelist.aEntry[i].iBlk, - pSnap->freelist.aEntry[i].iId - ); - } -#endif - - *ppCkpt = (void *)ckpt.aCkpt; - if( pnCkpt ) *pnCkpt = sizeof(u32)*iOut; - return rc; -} - - -/* -** Helper function for ckptImport(). -*/ -static void ckptNewSegment( - u32 *aIn, - int *piIn, - Segment *pSegment /* Populate this structure */ -){ - assert( pSegment->iFirst==0 && pSegment->iLastPg==0 ); - assert( pSegment->nSize==0 && pSegment->iRoot==0 ); - pSegment->iFirst = ckptGobble64(aIn, piIn); - pSegment->iLastPg = ckptGobble64(aIn, piIn); - pSegment->iRoot = ckptGobble64(aIn, piIn); - pSegment->nSize = ckptGobble64(aIn, piIn); - assert( pSegment->iFirst ); -} - -static int ckptSetupMerge(lsm_db *pDb, u32 *aInt, int *piIn, Level *pLevel){ - Merge *pMerge; /* Allocated Merge object */ - int nInput; /* Number of input segments in merge */ - int iIn = *piIn; /* Next value to read from aInt[] */ - int i; /* Iterator variable */ - int nByte; /* Number of bytes to allocate */ - - /* Allocate the Merge object. If malloc() fails, return LSM_NOMEM. */ - nInput = (int)aInt[iIn++]; - nByte = sizeof(Merge) + sizeof(MergeInput) * nInput; - pMerge = (Merge *)lsmMallocZero(pDb->pEnv, nByte); - if( !pMerge ) return LSM_NOMEM_BKPT; - pLevel->pMerge = pMerge; - - /* Populate the Merge object. */ - pMerge->aInput = (MergeInput *)&pMerge[1]; - pMerge->nInput = nInput; - pMerge->iOutputOff = -1; - pMerge->nSkip = (int)aInt[iIn++]; - for(i=0; i<nInput; i++){ - pMerge->aInput[i].iPg = ckptGobble64(aInt, &iIn); - pMerge->aInput[i].iCell = (int)aInt[iIn++]; - } - pMerge->splitkey.iPg = ckptGobble64(aInt, &iIn); - pMerge->splitkey.iCell = (int)aInt[iIn++]; - pMerge->iCurrentPtr = ckptGobble64(aInt, &iIn); - - /* Set *piIn and return LSM_OK. */ - *piIn = iIn; - return LSM_OK; -} - - -static int ckptLoadLevels( - lsm_db *pDb, - u32 *aIn, - int *piIn, - int nLevel, - Level **ppLevel -){ - int i; - int rc = LSM_OK; - Level *pRet = 0; - Level **ppNext; - int iIn = *piIn; - - ppNext = &pRet; - for(i=0; rc==LSM_OK && i<nLevel; i++){ - int iRight; - Level *pLevel; - - /* Allocate space for the Level structure and Level.apRight[] array */ - pLevel = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); - if( rc==LSM_OK ){ - pLevel->iAge = (u16)(aIn[iIn] & 0x0000FFFF); - pLevel->flags = (u16)((aIn[iIn]>>16) & 0x0000FFFF); - iIn++; - pLevel->nRight = aIn[iIn++]; - if( pLevel->nRight ){ - int nByte = sizeof(Segment) * pLevel->nRight; - pLevel->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - } - if( rc==LSM_OK ){ - *ppNext = pLevel; - ppNext = &pLevel->pNext; - - /* Allocate the main segment */ - ckptNewSegment(aIn, &iIn, &pLevel->lhs); - - /* Allocate each of the right-hand segments, if any */ - for(iRight=0; iRight<pLevel->nRight; iRight++){ - ckptNewSegment(aIn, &iIn, &pLevel->aRhs[iRight]); - } - - /* Set up the Merge object, if required */ - if( pLevel->nRight>0 ){ - rc = ckptSetupMerge(pDb, aIn, &iIn, pLevel); - } - } - } - } - - if( rc!=LSM_OK ){ - /* An OOM must have occurred. Free any level structures allocated and - ** return the error to the caller. */ - lsmSortedFreeLevel(pDb->pEnv, pRet); - pRet = 0; - } - - *ppLevel = pRet; - *piIn = iIn; - return rc; -} - - -int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal){ - int rc = LSM_OK; - if( nVal>0 ){ - u32 *aIn; - - aIn = lsmMallocRc(pDb->pEnv, nVal, &rc); - if( aIn ){ - Level *pLevel = 0; - Level *pParent; - - int nIn; - int nLevel; - int iIn = 1; - memcpy(aIn, pVal, nVal); - nIn = nVal / sizeof(u32); - - ckptChangeEndianness(aIn, nIn); - nLevel = aIn[0]; - rc = ckptLoadLevels(pDb, aIn, &iIn, nLevel, &pLevel); - lsmFree(pDb->pEnv, aIn); - assert( rc==LSM_OK || pLevel==0 ); - if( rc==LSM_OK ){ - pParent = lsmDbSnapshotLevel(pDb->pWorker); - assert( pParent ); - while( pParent->pNext ) pParent = pParent->pNext; - pParent->pNext = pLevel; - } - } - } - - return rc; -} - -/* -** Return the data for the LEVELS record. -** -** The size of the checkpoint that can be stored in the database header -** must not exceed 1024 32-bit integers. Normally, it does not. However, -** if it does, part of the checkpoint must be stored in the LSM. This -** routine returns that part. -*/ -int lsmCheckpointLevels( - lsm_db *pDb, /* Database handle */ - int nLevel, /* Number of levels to write to blob */ - void **paVal, /* OUT: Pointer to LEVELS blob */ - int *pnVal /* OUT: Size of LEVELS blob in bytes */ -){ - Level *p; /* Used to iterate through levels */ - int nAll= 0; - int rc; - int i; - int iOut; - CkptBuffer ckpt; - assert( nLevel>0 ); - - for(p=lsmDbSnapshotLevel(pDb->pWorker); p; p=p->pNext) nAll++; - - assert( nAll>nLevel ); - nAll -= nLevel; - for(p=lsmDbSnapshotLevel(pDb->pWorker); p && nAll>0; p=p->pNext) nAll--; - - memset(&ckpt, 0, sizeof(CkptBuffer)); - ckpt.pEnv = pDb->pEnv; - - ckptSetValue(&ckpt, 0, nLevel, &rc); - iOut = 1; - for(i=0; rc==LSM_OK && i<nLevel; i++){ - ckptExportLevel(p, &ckpt, &iOut, &rc); - p = p->pNext; - } - assert( rc!=LSM_OK || p==0 ); - - if( rc==LSM_OK ){ - ckptChangeEndianness(ckpt.aCkpt, iOut); - *paVal = (void *)ckpt.aCkpt; - *pnVal = iOut * sizeof(u32); - }else{ - *pnVal = 0; - *paVal = 0; - } - - return rc; -} - -/* -** Read the checkpoint id from meta-page pPg. -*/ -static i64 ckptLoadId(MetaPage *pPg){ - i64 ret = 0; - if( pPg ){ - int nData; - u8 *aData = lsmFsMetaPageData(pPg, &nData); - ret = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32) + - ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); - } - return ret; -} - -/* -** Return true if the buffer passed as an argument contains a valid -** checkpoint. -*/ -static int ckptChecksumOk(u32 *aCkpt){ - u32 nCkpt = aCkpt[CKPT_HDR_NCKPT]; - u32 cksum1; - u32 cksum2; - - if( nCkpt<CKPT_HDR_NCKPT || nCkpt>(LSM_META_RW_PAGE_SIZE)/sizeof(u32) ){ - return 0; - } - ckptChecksum(aCkpt, nCkpt, &cksum1, &cksum2); - return (cksum1==aCkpt[nCkpt-2] && cksum2==aCkpt[nCkpt-1]); -} - -/* -** Attempt to load a checkpoint from meta page iMeta. -** -** This function is a no-op if *pRc is set to any value other than LSM_OK -** when it is called. If an error occurs, *pRc is set to an LSM error code -** before returning. -** -** If no error occurs and the checkpoint is successfully loaded, copy it to -** ShmHeader.aSnap1[] and ShmHeader.aSnap2[], and set ShmHeader.iMetaPage -** to indicate its origin. In this case return 1. Or, if the checkpoint -** cannot be loaded (because the checksum does not compute), return 0. -*/ -static int ckptTryLoad(lsm_db *pDb, MetaPage *pPg, u32 iMeta, int *pRc){ - int bLoaded = 0; /* Return value */ - if( *pRc==LSM_OK ){ - int rc = LSM_OK; /* Error code */ - u32 *aCkpt = 0; /* Pointer to buffer containing checkpoint */ - u32 nCkpt; /* Number of elements in aCkpt[] */ - int nData; /* Bytes of data in aData[] */ - u8 *aData; /* Meta page data */ - - aData = lsmFsMetaPageData(pPg, &nData); - nCkpt = (u32)lsmGetU32(&aData[CKPT_HDR_NCKPT*sizeof(u32)]); - if( nCkpt<=nData/sizeof(u32) && nCkpt>CKPT_HDR_NCKPT ){ - aCkpt = (u32 *)lsmMallocRc(pDb->pEnv, nCkpt*sizeof(u32), &rc); - } - if( aCkpt ){ - memcpy(aCkpt, aData, nCkpt*sizeof(u32)); - ckptChangeEndianness(aCkpt, nCkpt); - if( ckptChecksumOk(aCkpt) ){ - ShmHeader *pShm = pDb->pShmhdr; - memcpy(pShm->aSnap1, aCkpt, nCkpt*sizeof(u32)); - memcpy(pShm->aSnap2, aCkpt, nCkpt*sizeof(u32)); - memcpy(pDb->aSnapshot, aCkpt, nCkpt*sizeof(u32)); - pShm->iMetaPage = iMeta; - bLoaded = 1; - } - } - - lsmFree(pDb->pEnv, aCkpt); - *pRc = rc; - } - return bLoaded; -} - -/* -** Initialize the shared-memory header with an empty snapshot. This function -** is called when no valid snapshot can be found in the database header. -*/ -static void ckptLoadEmpty(lsm_db *pDb){ - u32 aCkpt[] = { - 0, /* CKPT_HDR_ID_MSW */ - 10, /* CKPT_HDR_ID_LSW */ - 0, /* CKPT_HDR_NCKPT */ - LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ - 0, /* CKPT_HDR_NBLOCK */ - 0, /* CKPT_HDR_BLKSZ */ - 0, /* CKPT_HDR_NLEVEL */ - 0, /* CKPT_HDR_PGSZ */ - 0, /* CKPT_HDR_NWRITE */ - 0, 0, 1234, 5678, /* The log pointer and initial checksum */ - 0,0,0,0, 0,0,0,0, /* The append list */ - 0, /* The redirected block list */ - 0, /* The free block list */ - 0, 0 /* Space for checksum values */ - }; - u32 nCkpt = array_size(aCkpt); - ShmHeader *pShm = pDb->pShmhdr; - - aCkpt[CKPT_HDR_NCKPT] = nCkpt; - aCkpt[CKPT_HDR_BLKSZ] = pDb->nDfltBlksz; - aCkpt[CKPT_HDR_PGSZ] = pDb->nDfltPgsz; - ckptChecksum(aCkpt, array_size(aCkpt), &aCkpt[nCkpt-2], &aCkpt[nCkpt-1]); - - memcpy(pShm->aSnap1, aCkpt, nCkpt*sizeof(u32)); - memcpy(pShm->aSnap2, aCkpt, nCkpt*sizeof(u32)); - memcpy(pDb->aSnapshot, aCkpt, nCkpt*sizeof(u32)); -} - -/* -** This function is called as part of database recovery to initialize the -** ShmHeader.aSnap1[] and ShmHeader.aSnap2[] snapshots. -*/ -int lsmCheckpointRecover(lsm_db *pDb){ - int rc = LSM_OK; /* Return Code */ - i64 iId1; /* Id of checkpoint on meta-page 1 */ - i64 iId2; /* Id of checkpoint on meta-page 2 */ - int bLoaded = 0; /* True once checkpoint has been loaded */ - int cmp; /* True if (iId2>iId1) */ - MetaPage *apPg[2] = {0, 0}; /* Meta-pages 1 and 2 */ - - rc = lsmFsMetaPageGet(pDb->pFS, 0, 1, &apPg[0]); - if( rc==LSM_OK ) rc = lsmFsMetaPageGet(pDb->pFS, 0, 2, &apPg[1]); - - iId1 = ckptLoadId(apPg[0]); - iId2 = ckptLoadId(apPg[1]); - cmp = (iId2 > iId1); - bLoaded = ckptTryLoad(pDb, apPg[cmp?1:0], (cmp?2:1), &rc); - if( bLoaded==0 ){ - bLoaded = ckptTryLoad(pDb, apPg[cmp?0:1], (cmp?1:2), &rc); - } - - /* The database does not contain a valid checkpoint. Initialize the shared - ** memory header with an empty checkpoint. */ - if( bLoaded==0 ){ - ckptLoadEmpty(pDb); - } - - lsmFsMetaPageRelease(apPg[0]); - lsmFsMetaPageRelease(apPg[1]); - - return rc; -} - -/* -** Store the snapshot in pDb->aSnapshot[] in meta-page iMeta. -*/ -int lsmCheckpointStore(lsm_db *pDb, int iMeta){ - MetaPage *pPg = 0; - int rc; - - assert( iMeta==1 || iMeta==2 ); - rc = lsmFsMetaPageGet(pDb->pFS, 1, iMeta, &pPg); - if( rc==LSM_OK ){ - u8 *aData; - int nData; - int nCkpt; - - nCkpt = (int)pDb->aSnapshot[CKPT_HDR_NCKPT]; - aData = lsmFsMetaPageData(pPg, &nData); - memcpy(aData, pDb->aSnapshot, nCkpt*sizeof(u32)); - ckptChangeEndianness((u32 *)aData, nCkpt); - rc = lsmFsMetaPageRelease(pPg); - } - - return rc; -} - -/* -** Copy the current client snapshot from shared-memory to pDb->aSnapshot[]. -*/ -int lsmCheckpointLoad(lsm_db *pDb, int *piRead){ - int nRem = LSM_ATTEMPTS_BEFORE_PROTOCOL; - ShmHeader *pShm = pDb->pShmhdr; - while( (nRem--)>0 ){ - int nInt; - - nInt = pShm->aSnap1[CKPT_HDR_NCKPT]; - if( nInt<=(LSM_META_RW_PAGE_SIZE / sizeof(u32)) ){ - memcpy(pDb->aSnapshot, pShm->aSnap1, nInt*sizeof(u32)); - if( ckptChecksumOk(pDb->aSnapshot) ){ - if( piRead ) *piRead = 1; - return LSM_OK; - } - } - - nInt = pShm->aSnap2[CKPT_HDR_NCKPT]; - if( nInt<=(LSM_META_RW_PAGE_SIZE / sizeof(u32)) ){ - memcpy(pDb->aSnapshot, pShm->aSnap2, nInt*sizeof(u32)); - if( ckptChecksumOk(pDb->aSnapshot) ){ - if( piRead ) *piRead = 2; - return LSM_OK; - } - } - - lsmShmBarrier(pDb); - } - return LSM_PROTOCOL_BKPT; -} - -int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId){ - int rc; - - assert( db->pClient==0 && db->pWorker==0 ); - rc = lsmCheckpointLoad(db, 0); - if( rc==LSM_OK ){ - *piCmpId = db->aSnapshot[CKPT_HDR_CMPID]; - } - - return rc; -} - -int lsmCheckpointLoadOk(lsm_db *pDb, int iSnap){ - u32 *aShm; - assert( iSnap==1 || iSnap==2 ); - aShm = (iSnap==1) ? pDb->pShmhdr->aSnap1 : pDb->pShmhdr->aSnap2; - return (lsmCheckpointId(pDb->aSnapshot, 0)==lsmCheckpointId(aShm, 0) ); -} - -int lsmCheckpointClientCacheOk(lsm_db *pDb){ - return ( pDb->pClient - && pDb->pClient->iId==lsmCheckpointId(pDb->aSnapshot, 0) - && pDb->pClient->iId==lsmCheckpointId(pDb->pShmhdr->aSnap1, 0) - && pDb->pClient->iId==lsmCheckpointId(pDb->pShmhdr->aSnap2, 0) - ); -} - -int lsmCheckpointLoadWorker(lsm_db *pDb){ - int rc; - ShmHeader *pShm = pDb->pShmhdr; - int nInt1; - int nInt2; - - /* Must be holding the WORKER lock to do this. Or DMS2. */ - assert( - lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) - || lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) - ); - - /* Check that the two snapshots match. If not, repair them. */ - nInt1 = pShm->aSnap1[CKPT_HDR_NCKPT]; - nInt2 = pShm->aSnap2[CKPT_HDR_NCKPT]; - if( nInt1!=nInt2 || memcmp(pShm->aSnap1, pShm->aSnap2, nInt2*sizeof(u32)) ){ - if( ckptChecksumOk(pShm->aSnap1) ){ - memcpy(pShm->aSnap2, pShm->aSnap1, sizeof(u32)*nInt1); - }else if( ckptChecksumOk(pShm->aSnap2) ){ - memcpy(pShm->aSnap1, pShm->aSnap2, sizeof(u32)*nInt2); - }else{ - return LSM_PROTOCOL_BKPT; - } - } - - rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); - if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; - - if( rc==LSM_OK ){ - rc = lsmCheckCompressionId(pDb, pDb->pWorker->iCmpId); - } - -#if 0 - assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); -#endif - return rc; -} - -int lsmCheckpointDeserialize( - lsm_db *pDb, - int bInclFreelist, /* If true, deserialize free-list */ - u32 *aCkpt, - Snapshot **ppSnap -){ - int rc = LSM_OK; - Snapshot *pNew; - - pNew = (Snapshot *)lsmMallocZeroRc(pDb->pEnv, sizeof(Snapshot), &rc); - if( rc==LSM_OK ){ - Level *pLvl; - int nFree; - int i; - int nLevel = (int)aCkpt[CKPT_HDR_NLEVEL]; - int iIn = CKPT_HDR_SIZE + CKPT_APPENDLIST_SIZE + CKPT_LOGPTR_SIZE; - - pNew->iId = lsmCheckpointId(aCkpt, 0); - pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; - pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; - rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); - pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); - pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; - - /* Make a copy of the append-list */ - for(i=0; i<LSM_APPLIST_SZ; i++){ - u32 *a = &aCkpt[CKPT_HDR_SIZE + CKPT_LOGPTR_SIZE + i*2]; - pNew->aiAppend[i] = ckptRead64(a); - } - - /* Read the block-redirect list */ - pNew->redirect.n = aCkpt[iIn++]; - if( pNew->redirect.n ){ - pNew->redirect.a = lsmMallocZeroRc(pDb->pEnv, - (sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS), &rc - ); - if( rc==LSM_OK ){ - for(i=0; i<pNew->redirect.n; i++){ - pNew->redirect.a[i].iFrom = aCkpt[iIn++]; - pNew->redirect.a[i].iTo = aCkpt[iIn++]; - } - } - for(pLvl=pNew->pLevel; pLvl->pNext; pLvl=pLvl->pNext); - if( pLvl->nRight ){ - pLvl->aRhs[pLvl->nRight-1].pRedirect = &pNew->redirect; - }else{ - pLvl->lhs.pRedirect = &pNew->redirect; - } - } - - /* Copy the free-list */ - if( rc==LSM_OK && bInclFreelist ){ - nFree = aCkpt[iIn++]; - if( nFree ){ - pNew->freelist.aEntry = (FreelistEntry *)lsmMallocZeroRc( - pDb->pEnv, sizeof(FreelistEntry)*nFree, &rc - ); - if( rc==LSM_OK ){ - int j; - for(j=0; j<nFree; j++){ - FreelistEntry *p = &pNew->freelist.aEntry[j]; - p->iBlk = aCkpt[iIn++]; - p->iId = ((i64)(aCkpt[iIn])<<32) + aCkpt[iIn+1]; - iIn += 2; - } - pNew->freelist.nEntry = pNew->freelist.nAlloc = nFree; - } - } - } - } - - if( rc!=LSM_OK ){ - lsmFreeSnapshot(pDb->pEnv, pNew); - pNew = 0; - } - - *ppSnap = pNew; - return rc; -} - -/* -** Connection pDb must be the worker connection in order to call this -** function. It returns true if the database already contains the maximum -** number of levels or false otherwise. -** -** This is used when flushing the in-memory tree to disk. If the database -** is already full, then the caller should invoke lsm_work() or similar -** until it is not full before creating a new level by flushing the in-memory -** tree to disk. Limiting the number of levels in the database ensures that -** the records describing them always fit within the checkpoint blob. -*/ -int lsmDatabaseFull(lsm_db *pDb){ - Level *p; - int nRhs = 0; - - assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) ); - assert( pDb->pWorker ); - - for(p=pDb->pWorker->pLevel; p; p=p->pNext){ - nRhs += (p->nRight ? p->nRight : 1); - } - - return (nRhs >= LSM_MAX_RHS_SEGMENTS); -} - -/* -** The connection passed as the only argument is currently the worker -** connection. Some work has been performed on the database by the connection, -** but no new snapshot has been written into shared memory. -** -** This function updates the shared-memory worker and client snapshots with -** the new snapshot produced by the work performed by pDb. -** -** If successful, LSM_OK is returned. Otherwise, if an error occurs, an LSM -** error code is returned. -*/ -int lsmCheckpointSaveWorker(lsm_db *pDb, int bFlush){ - Snapshot *pSnap = pDb->pWorker; - ShmHeader *pShm = pDb->pShmhdr; - void *p = 0; - int n = 0; - int rc; - - pSnap->iId++; - rc = ckptExportSnapshot(pDb, bFlush, pSnap->iId, 1, &p, &n); - if( rc!=LSM_OK ) return rc; - assert( ckptChecksumOk((u32 *)p) ); - - assert( n<=LSM_META_RW_PAGE_SIZE ); - memcpy(pShm->aSnap2, p, n); - lsmShmBarrier(pDb); - memcpy(pShm->aSnap1, p, n); - lsmFree(pDb->pEnv, p); - - /* assert( lsmFsIntegrityCheck(pDb) ); */ - return LSM_OK; -} - -/* -** This function is used to determine the snapshot-id of the most recently -** checkpointed snapshot. Variable ShmHeader.iMetaPage indicates which of -** the two meta-pages said snapshot resides on (if any). -** -** If successful, this function loads the snapshot from the meta-page, -** verifies its checksum and sets *piId to the snapshot-id before returning -** LSM_OK. Or, if the checksum attempt fails, *piId is set to zero and -** LSM_OK returned. If an error occurs, an LSM error code is returned and -** the final value of *piId is undefined. -*/ -int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite){ - int rc = LSM_OK; - MetaPage *pPg; - u32 iMeta; - - iMeta = pDb->pShmhdr->iMetaPage; - if( iMeta==1 || iMeta==2 ){ - rc = lsmFsMetaPageGet(pDb->pFS, 0, iMeta, &pPg); - if( rc==LSM_OK ){ - int nCkpt; - int nData; - u8 *aData; - - aData = lsmFsMetaPageData(pPg, &nData); - assert( nData==LSM_META_RW_PAGE_SIZE ); - nCkpt = lsmGetU32(&aData[CKPT_HDR_NCKPT*sizeof(u32)]); - if( nCkpt<(LSM_META_RW_PAGE_SIZE/sizeof(u32)) ){ - u32 *aCopy = lsmMallocRc(pDb->pEnv, sizeof(u32) * nCkpt, &rc); - if( aCopy ){ - memcpy(aCopy, aData, nCkpt*sizeof(u32)); - ckptChangeEndianness(aCopy, nCkpt); - if( ckptChecksumOk(aCopy) ){ - if( piId ) *piId = lsmCheckpointId(aCopy, 0); - if( piLog ) *piLog = (lsmCheckpointLogOffset(aCopy) >> 1); - if( pnWrite ) *pnWrite = aCopy[CKPT_HDR_NWRITE]; - } - lsmFree(pDb->pEnv, aCopy); - } - } - lsmFsMetaPageRelease(pPg); - } - } - - if( (iMeta!=1 && iMeta!=2) || rc!=LSM_OK || pDb->pShmhdr->iMetaPage!=iMeta ){ - if( piId ) *piId = 0; - if( piLog ) *piLog = 0; - if( pnWrite ) *pnWrite = 0; - } - return rc; -} - -/* -** Return the checkpoint-id of the checkpoint array passed as the first -** argument to this function. If the second argument is true, then assume -** that the checkpoint is made up of 32-bit big-endian integers. If it -** is false, assume that the integers are in machine byte order. -*/ -i64 lsmCheckpointId(u32 *aCkpt, int bDisk){ - i64 iId; - if( bDisk ){ - u8 *aData = (u8 *)aCkpt; - iId = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32); - iId += ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); - }else{ - iId = ((i64)aCkpt[CKPT_HDR_ID_MSW] << 32) + (i64)aCkpt[CKPT_HDR_ID_LSW]; - } - return iId; -} - -u32 lsmCheckpointNBlock(u32 *aCkpt){ - return aCkpt[CKPT_HDR_NBLOCK]; -} - -u32 lsmCheckpointNWrite(u32 *aCkpt, int bDisk){ - if( bDisk ){ - return lsmGetU32((u8 *)&aCkpt[CKPT_HDR_NWRITE]); - }else{ - return aCkpt[CKPT_HDR_NWRITE]; - } -} - -i64 lsmCheckpointLogOffset(u32 *aCkpt){ - return ((i64)aCkpt[CKPT_HDR_LO_MSW] << 32) + (i64)aCkpt[CKPT_HDR_LO_LSW]; -} - -int lsmCheckpointPgsz(u32 *aCkpt){ return (int)aCkpt[CKPT_HDR_PGSZ]; } - -int lsmCheckpointBlksz(u32 *aCkpt){ return (int)aCkpt[CKPT_HDR_BLKSZ]; } - -void lsmCheckpointLogoffset( - u32 *aCkpt, - DbLog *pLog -){ - pLog->aRegion[2].iStart = (lsmCheckpointLogOffset(aCkpt) >> 1); - - pLog->cksum0 = aCkpt[CKPT_HDR_LO_CKSUM1]; - pLog->cksum1 = aCkpt[CKPT_HDR_LO_CKSUM2]; - pLog->iSnapshotId = lsmCheckpointId(aCkpt, 0); -} - -void lsmCheckpointZeroLogoffset(lsm_db *pDb){ - u32 nCkpt; - - nCkpt = pDb->aSnapshot[CKPT_HDR_NCKPT]; - assert( nCkpt>CKPT_HDR_NCKPT ); - assert( nCkpt==pDb->pShmhdr->aSnap1[CKPT_HDR_NCKPT] ); - assert( 0==memcmp(pDb->aSnapshot, pDb->pShmhdr->aSnap1, nCkpt*sizeof(u32)) ); - assert( 0==memcmp(pDb->aSnapshot, pDb->pShmhdr->aSnap2, nCkpt*sizeof(u32)) ); - - pDb->aSnapshot[CKPT_HDR_LO_MSW] = 0; - pDb->aSnapshot[CKPT_HDR_LO_LSW] = 0; - ckptChecksum(pDb->aSnapshot, nCkpt, - &pDb->aSnapshot[nCkpt-2], &pDb->aSnapshot[nCkpt-1] - ); - - memcpy(pDb->pShmhdr->aSnap1, pDb->aSnapshot, nCkpt*sizeof(u32)); - memcpy(pDb->pShmhdr->aSnap2, pDb->aSnapshot, nCkpt*sizeof(u32)); -} - -/* -** Set the output variable to the number of KB of data written into the -** database file since the most recent checkpoint. -*/ -int lsmCheckpointSize(lsm_db *db, int *pnKB){ - int rc = LSM_OK; - u32 nSynced; - - /* Set nSynced to the number of pages that had been written when the - ** database was last checkpointed. */ - rc = lsmCheckpointSynced(db, 0, 0, &nSynced); - - if( rc==LSM_OK ){ - u32 nPgsz = db->pShmhdr->aSnap1[CKPT_HDR_PGSZ]; - u32 nWrite = db->pShmhdr->aSnap1[CKPT_HDR_NWRITE]; - *pnKB = (int)(( ((i64)(nWrite - nSynced) * nPgsz) + 1023) / 1024); - } - - return rc; -} diff --git a/ext/lsm1/lsm_file.c b/ext/lsm1/lsm_file.c deleted file mode 100644 index 9f4144618..000000000 --- a/ext/lsm1/lsm_file.c +++ /dev/null @@ -1,3311 +0,0 @@ -/* -** 2011-08-26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** NORMAL DATABASE FILE FORMAT -** -** The following database file format concepts are used by the code in -** this file to read and write the database file. -** -** Pages: -** -** A database file is divided into pages. The first 8KB of the file consists -** of two 4KB meta-pages. The meta-page size is not configurable. The -** remainder of the file is made up of database pages. The default database -** page size is 4KB. Database pages are aligned to page-size boundaries, -** so if the database page size is larger than 8KB there is a gap between -** the end of the meta pages and the start of the database pages. -** -** Database pages are numbered based on their position in the file. Page N -** begins at byte offset ((N-1)*pgsz). This means that page 1 does not -** exist - since it would always overlap with the meta pages. If the -** page-size is (say) 512 bytes, then the first usable page in the database -** is page 33. -** -** It is assumed that the first two meta pages and the data that follows -** them are located on different disk sectors. So that if a power failure -** while writing to a meta page there is no risk of damage to the other -** meta page or any other part of the database file. TODO: This may need -** to be revisited. -** -** Blocks: -** -** The database file is also divided into blocks. The default block size is -** 1MB. When writing to the database file, an attempt is made to write data -** in contiguous block-sized chunks. -** -** The first and last page on each block are special in that they are 4 -** bytes smaller than all other pages. This is because the last four bytes -** of space on the first and last pages of each block are reserved for -** pointers to other blocks (i.e. a 32-bit block number). -** -** Runs: -** -** A run is a sequence of pages that the upper layer uses to store a -** sorted array of database keys (and accompanying data - values, FC -** pointers and so on). Given a page within a run, it is possible to -** navigate to the next page in the run as follows: -** -** a) if the current page is not the last in a block, the next page -** in the run is located immediately after the current page, OR -** -** b) if the current page is the last page in a block, the next page -** in the run is the first page on the block identified by the -** block pointer stored in the last 4 bytes of the current block. -** -** It is possible to navigate to the previous page in a similar fashion, -** using the block pointer embedded in the last 4 bytes of the first page -** of each block as required. -** -** The upper layer is responsible for identifying by page number the -** first and last page of any run that it needs to navigate - there are -** no "end-of-run" markers stored or identified by this layer. This is -** necessary as clients reading different database snapshots may access -** different subsets of a run. -** -** THE LOG FILE -** -** This file opens and closes the log file. But it does not contain any -** logic related to the log file format. Instead, it exports the following -** functions that are used by the code in lsm_log.c to read and write the -** log file: -** -** lsmFsOpenLog -** lsmFsWriteLog -** lsmFsSyncLog -** lsmFsReadLog -** lsmFsTruncateLog -** lsmFsCloseAndDeleteLog -** -** COMPRESSED DATABASE FILE FORMAT -** -** The compressed database file format is very similar to the normal format. -** The file still begins with two 4KB meta-pages (which are never compressed). -** It is still divided into blocks. -** -** The first and last four bytes of each block are reserved for 32-bit -** pointer values. Similar to the way four bytes are carved from the end of -** the first and last page of each block in uncompressed databases. From -** the point of view of the upper layer, all pages are the same size - this -** is different from the uncompressed format where the first and last pages -** on each block are 4 bytes smaller than the others. -** -** Pages are stored in variable length compressed form, as follows: -** -** * 3-byte size field containing the size of the compressed page image -** in bytes. The most significant bit of each byte of the size field -** is always set. The remaining 7 bits are used to store a 21-bit -** integer value (in big-endian order - the first byte in the field -** contains the most significant 7 bits). Since the maximum allowed -** size of a compressed page image is (2^17 - 1) bytes, there are -** actually 4 unused bits in the size field. -** -** In other words, if the size of the compressed page image is nSz, -** the header can be serialized as follows: -** -** u8 aHdr[3] -** aHdr[0] = 0x80 | (u8)(nSz >> 14); -** aHdr[1] = 0x80 | (u8)(nSz >> 7); -** aHdr[2] = 0x80 | (u8)(nSz >> 0); -** -** * Compressed page image. -** -** * A second copy of the 3-byte record header. -** -** A page number is a byte offset into the database file. So the smallest -** possible page number is 8192 (immediately after the two meta-pages). -** The first and root page of a segment are identified by a page number -** corresponding to the byte offset of the first byte in the corresponding -** page record. The last page of a segment is identified by the byte offset -** of the last byte in its record. -** -** Unlike uncompressed pages, compressed page records may span blocks. -** -** Sometimes, in order to avoid touching sectors that contain synced data -** when writing, it is necessary to insert unused space between compressed -** page records. This can be done as follows: -** -** * For less than 6 bytes of empty space, the first and last byte -** of the free space contain the total number of free bytes. For -** example: -** -** Block of 4 free bytes: 0x04 0x?? 0x?? 0x04 -** Block of 2 free bytes: 0x02 0x02 -** A single free byte: 0x01 -** -** * For 6 or more bytes of empty space, a record similar to a -** compressed page record is added to the segment. A padding record -** is distinguished from a compressed page record by the most -** significant bit of the second byte of the size field, which is -** cleared instead of set. -*/ -#include "lsmInt.h" - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - -/* -** File-system object. Each database connection allocates a single instance -** of the following structure. It is used for all access to the database and -** log files. -** -** The database file may be accessed via two methods - using mmap() or using -** read() and write() calls. In the general case both methods are used - a -** prefix of the file is mapped into memory and the remainder accessed using -** read() and write(). This is helpful when accessing very large files (or -** files that may grow very large during the lifetime of a database -** connection) on systems with 32-bit address spaces. However, it also requires -** that this object manage two distinct types of Page objects simultaneously - -** those that carry pointers to the mapped file and those that carry arrays -** populated by read() calls. -** -** pFree: -** The head of a singly-linked list that containing currently unused Page -** structures suitable for use as mmap-page handles. Connected by the -** Page.pFreeNext pointers. -** -** pMapped: -** The head of a singly-linked list that contains all pages that currently -** carry pointers to the mapped region. This is used if the region is -** every remapped - the pointers carried by existing pages can be adjusted -** to account for the remapping. Connected by the Page.pMappedNext pointers. -** -** pWaiting: -** When the upper layer wishes to append a new b-tree page to a segment, -** it allocates a Page object that carries a malloc'd block of memory - -** regardless of the mmap-related configuration. The page is not assigned -** a page number at first. When the upper layer has finished constructing -** the page contents, it calls lsmFsPagePersist() to assign a page number -** to it. At this point it is likely that N pages have been written to the -** segment, the (N+1)th page is still outstanding and the b-tree page is -** assigned page number (N+2). To avoid writing page (N+2) before page -** (N+1), the recently completed b-tree page is held in the singly linked -** list headed by pWaiting until page (N+1) has been written. -** -** Function lsmFsFlushWaiting() is responsible for eventually writing -** waiting pages to disk. -** -** apHash/nHash: -** Hash table used to store all Page objects that carry malloc'd arrays, -** except those b-tree pages that have not yet been assigned page numbers. -** Once they have been assigned page numbers - they are added to this -** hash table. -** -** Hash table overflow chains are connected using the Page.pHashNext -** pointers. -** -** pLruFirst, pLruLast: -** The first and last entries in a doubly-linked list of pages. This -** list contains all pages with malloc'd data that are present in the -** hash table and have a ref-count of zero. -*/ -struct FileSystem { - lsm_db *pDb; /* Database handle that owns this object */ - lsm_env *pEnv; /* Environment pointer */ - char *zDb; /* Database file name */ - char *zLog; /* Database file name */ - int nMetasize; /* Size of meta pages in bytes */ - int nMetaRwSize; /* Read/written size of meta pages in bytes */ - i64 nPagesize; /* Database page-size in bytes */ - i64 nBlocksize; /* Database block-size in bytes */ - - /* r/w file descriptors for both files. */ - LsmFile *pLsmFile; /* Used after lsm_close() to link into list */ - lsm_file *fdDb; /* Database file */ - lsm_file *fdLog; /* Log file */ - int szSector; /* Database file sector size */ - - /* If this is a compressed database, a pointer to the compression methods. - ** For an uncompressed database, a NULL pointer. */ - lsm_compress *pCompress; - u8 *aIBuffer; /* Buffer to compress to */ - u8 *aOBuffer; /* Buffer to uncompress from */ - int nBuffer; /* Allocated size of above buffers in bytes */ - - /* mmap() page related things */ - i64 nMapLimit; /* Maximum bytes of file to map */ - void *pMap; /* Current mapping of database file */ - i64 nMap; /* Bytes mapped at pMap */ - Page *pFree; /* Unused Page structures */ - Page *pMapped; /* List of Page structs that point to pMap */ - - /* Page cache parameters for non-mmap() pages */ - int nCacheMax; /* Configured cache size (in pages) */ - int nCacheAlloc; /* Current cache size (in pages) */ - Page *pLruFirst; /* Head of the LRU list */ - Page *pLruLast; /* Tail of the LRU list */ - int nHash; /* Number of hash slots in hash table */ - Page **apHash; /* nHash Hash slots */ - Page *pWaiting; /* b-tree pages waiting to be written */ - - /* Statistics */ - int nOut; /* Number of outstanding pages */ - int nWrite; /* Total number of pages written */ - int nRead; /* Total number of pages read */ -}; - -/* -** Database page handle. -** -** pSeg: -** When lsmFsSortedAppend() is called on a compressed database, the new -** page is not assigned a page number or location in the database file -** immediately. Instead, these are assigned by the lsmFsPagePersist() call -** right before it writes the compressed page image to disk. -** -** The lsmFsSortedAppend() function sets the pSeg pointer to point to the -** segment that the new page will be a part of. It is unset by -** lsmFsPagePersist() after the page is written to disk. -*/ -struct Page { - u8 *aData; /* Buffer containing page data */ - int nData; /* Bytes of usable data at aData[] */ - LsmPgno iPg; /* Page number */ - int nRef; /* Number of outstanding references */ - int flags; /* Combination of PAGE_XXX flags */ - Page *pHashNext; /* Next page in hash table slot */ - Page *pLruNext; /* Next page in LRU list */ - Page *pLruPrev; /* Previous page in LRU list */ - FileSystem *pFS; /* File system that owns this page */ - - /* Only used in compressed database mode: */ - int nCompress; /* Compressed size (or 0 for uncomp. db) */ - int nCompressPrev; /* Compressed size of prev page */ - Segment *pSeg; /* Segment this page will be written to */ - - /* Pointers for singly linked lists */ - Page *pWaitingNext; /* Next page in FileSystem.pWaiting list */ - Page *pFreeNext; /* Next page in FileSystem.pFree list */ - Page *pMappedNext; /* Next page in FileSystem.pMapped list */ -}; - -/* -** Meta-data page handle. There are two meta-data pages at the start of -** the database file, each FileSystem.nMetasize bytes in size. -*/ -struct MetaPage { - int iPg; /* Either 1 or 2 */ - int bWrite; /* Write back to db file on release */ - u8 *aData; /* Pointer to buffer */ - FileSystem *pFS; /* FileSystem that owns this page */ -}; - -/* -** Values for LsmPage.flags -*/ -#define PAGE_DIRTY 0x00000001 /* Set if page is dirty */ -#define PAGE_FREE 0x00000002 /* Set if Page.aData requires lsmFree() */ -#define PAGE_HASPREV 0x00000004 /* Set if page is first on uncomp. block */ - -/* -** Number of pgsz byte pages omitted from the start of block 1. The start -** of block 1 contains two 4096 byte meta pages (8192 bytes in total). -*/ -#define BLOCK1_HDR_SIZE(pgsz) LSM_MAX(1, 8192/(pgsz)) - -/* -** If NDEBUG is not defined, set a breakpoint in function lsmIoerrBkpt() -** to catch IO errors (any error returned by a VFS method). -*/ -#ifndef NDEBUG -static void lsmIoerrBkpt(void){ - static int nErr = 0; - nErr++; -} -static int IOERR_WRAPPER(int rc){ - if( rc!=LSM_OK ) lsmIoerrBkpt(); - return rc; -} -#else -# define IOERR_WRAPPER(rc) (rc) -#endif - -#ifdef NDEBUG -# define assert_lists_are_ok(x) -#else -static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash); - -static void assert_lists_are_ok(FileSystem *pFS){ -#if 0 - Page *p; - - assert( pFS->nMapLimit>=0 ); - - /* Check that all pages in the LRU list have nRef==0, pointers to buffers - ** in heap memory, and corresponding entries in the hash table. */ - for(p=pFS->pLruFirst; p; p=p->pLruNext){ - assert( p==pFS->pLruFirst || p->pLruPrev!=0 ); - assert( p==pFS->pLruLast || p->pLruNext!=0 ); - assert( p->pLruPrev==0 || p->pLruPrev->pLruNext==p ); - assert( p->pLruNext==0 || p->pLruNext->pLruPrev==p ); - assert( p->nRef==0 ); - assert( p->flags & PAGE_FREE ); - assert( p==fsPageFindInHash(pFS, p->iPg, 0) ); - } -#endif -} -#endif - -/* -** Wrappers around the VFS methods of the lsm_env object: -** -** lsmEnvOpen() -** lsmEnvRead() -** lsmEnvWrite() -** lsmEnvSync() -** lsmEnvSectorSize() -** lsmEnvClose() -** lsmEnvTruncate() -** lsmEnvUnlink() -** lsmEnvRemap() -*/ -int lsmEnvOpen(lsm_env *pEnv, const char *zFile, int flags, lsm_file **ppNew){ - return pEnv->xOpen(pEnv, zFile, flags, ppNew); -} - -static int lsmEnvRead( - lsm_env *pEnv, - lsm_file *pFile, - lsm_i64 iOff, - void *pRead, - int nRead -){ - return IOERR_WRAPPER( pEnv->xRead(pFile, iOff, pRead, nRead) ); -} - -static int lsmEnvWrite( - lsm_env *pEnv, - lsm_file *pFile, - lsm_i64 iOff, - const void *pWrite, - int nWrite -){ - return IOERR_WRAPPER( pEnv->xWrite(pFile, iOff, (void *)pWrite, nWrite) ); -} - -static int lsmEnvSync(lsm_env *pEnv, lsm_file *pFile){ - return IOERR_WRAPPER( pEnv->xSync(pFile) ); -} - -static int lsmEnvSectorSize(lsm_env *pEnv, lsm_file *pFile){ - return pEnv->xSectorSize(pFile); -} - -int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile){ - return IOERR_WRAPPER( pEnv->xClose(pFile) ); -} - -static int lsmEnvTruncate(lsm_env *pEnv, lsm_file *pFile, lsm_i64 nByte){ - return IOERR_WRAPPER( pEnv->xTruncate(pFile, nByte) ); -} - -static int lsmEnvUnlink(lsm_env *pEnv, const char *zDel){ - return IOERR_WRAPPER( pEnv->xUnlink(pEnv, zDel) ); -} - -static int lsmEnvRemap( - lsm_env *pEnv, - lsm_file *pFile, - i64 szMin, - void **ppMap, - i64 *pszMap -){ - return pEnv->xRemap(pFile, szMin, ppMap, pszMap); -} - -int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock){ - if( pFile==0 ) return LSM_OK; - return pEnv->xLock(pFile, iLock, eLock); -} - -int lsmEnvTestLock( - lsm_env *pEnv, - lsm_file *pFile, - int iLock, - int nLock, - int eLock -){ - return pEnv->xTestLock(pFile, iLock, nLock, eLock); -} - -int lsmEnvShmMap( - lsm_env *pEnv, - lsm_file *pFile, - int iChunk, - int sz, - void **ppOut -){ - return pEnv->xShmMap(pFile, iChunk, sz, ppOut); -} - -void lsmEnvShmBarrier(lsm_env *pEnv){ - pEnv->xShmBarrier(); -} - -void lsmEnvShmUnmap(lsm_env *pEnv, lsm_file *pFile, int bDel){ - pEnv->xShmUnmap(pFile, bDel); -} - -void lsmEnvSleep(lsm_env *pEnv, int nUs){ - pEnv->xSleep(pEnv, nUs); -} - - -/* -** Write the contents of string buffer pStr into the log file, starting at -** offset iOff. -*/ -int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr){ - assert( pFS->fdLog ); - return lsmEnvWrite(pFS->pEnv, pFS->fdLog, iOff, pStr->z, pStr->n); -} - -/* -** fsync() the log file. -*/ -int lsmFsSyncLog(FileSystem *pFS){ - assert( pFS->fdLog ); - return lsmEnvSync(pFS->pEnv, pFS->fdLog); -} - -/* -** Read nRead bytes of data starting at offset iOff of the log file. Append -** the results to string buffer pStr. -*/ -int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr){ - int rc; /* Return code */ - assert( pFS->fdLog ); - rc = lsmStringExtend(pStr, nRead); - if( rc==LSM_OK ){ - rc = lsmEnvRead(pFS->pEnv, pFS->fdLog, iOff, &pStr->z[pStr->n], nRead); - pStr->n += nRead; - } - return rc; -} - -/* -** Truncate the log file to nByte bytes in size. -*/ -int lsmFsTruncateLog(FileSystem *pFS, i64 nByte){ - if( pFS->fdLog==0 ) return LSM_OK; - return lsmEnvTruncate(pFS->pEnv, pFS->fdLog, nByte); -} - -/* -** Truncate the db file to nByte bytes in size. -*/ -int lsmFsTruncateDb(FileSystem *pFS, i64 nByte){ - if( pFS->fdDb==0 ) return LSM_OK; - return lsmEnvTruncate(pFS->pEnv, pFS->fdDb, nByte); -} - -/* -** Close the log file. Then delete it from the file-system. This function -** is called during database shutdown only. -*/ -int lsmFsCloseAndDeleteLog(FileSystem *pFS){ - char *zDel; - - if( pFS->fdLog ){ - lsmEnvClose(pFS->pEnv, pFS->fdLog ); - pFS->fdLog = 0; - } - - zDel = lsmMallocPrintf(pFS->pEnv, "%s-log", pFS->zDb); - if( zDel ){ - lsmEnvUnlink(pFS->pEnv, zDel); - lsmFree(pFS->pEnv, zDel); - } - return LSM_OK; -} - -/* -** Return true if page iReal of the database should be accessed using mmap. -** False otherwise. -*/ -static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){ - return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit); -} - -/* -** Given that there are currently nHash slots in the hash table, return -** the hash key for file iFile, page iPg. -*/ -static int fsHashKey(int nHash, LsmPgno iPg){ - return (iPg % nHash); -} - -/* -** This is a helper function for lsmFsOpen(). It opens a single file on -** disk (either the database or log file). -*/ -static lsm_file *fsOpenFile( - FileSystem *pFS, /* File system object */ - int bReadonly, /* True to open this file read-only */ - int bLog, /* True for log, false for db */ - int *pRc /* IN/OUT: Error code */ -){ - lsm_file *pFile = 0; - if( *pRc==LSM_OK ){ - int flags = (bReadonly ? LSM_OPEN_READONLY : 0); - const char *zPath = (bLog ? pFS->zLog : pFS->zDb); - - *pRc = lsmEnvOpen(pFS->pEnv, zPath, flags, &pFile); - } - return pFile; -} - -/* -** If it is not already open, this function opens the log file. It returns -** LSM_OK if successful (or if the log file was already open) or an LSM -** error code otherwise. -** -** The log file must be opened before any of the following may be called: -** -** lsmFsWriteLog -** lsmFsSyncLog -** lsmFsReadLog -*/ -int lsmFsOpenLog(lsm_db *db, int *pbOpen){ - int rc = LSM_OK; - FileSystem *pFS = db->pFS; - - if( 0==pFS->fdLog ){ - pFS->fdLog = fsOpenFile(pFS, db->bReadonly, 1, &rc); - - if( rc==LSM_IOERR_NOENT && db->bReadonly ){ - rc = LSM_OK; - } - } - - if( pbOpen ) *pbOpen = (pFS->fdLog!=0); - return rc; -} - -/* -** Close the log file, if it is open. -*/ -void lsmFsCloseLog(lsm_db *db){ - FileSystem *pFS = db->pFS; - if( pFS->fdLog ){ - lsmEnvClose(pFS->pEnv, pFS->fdLog); - pFS->fdLog = 0; - } -} - -/* -** Open a connection to a database stored within the file-system. -** -** If parameter bReadonly is true, then open a read-only file-descriptor -** on the database file. It is possible that bReadonly will be false even -** if the user requested that pDb be opened read-only. This is because the -** file-descriptor may later on be recycled by a read-write connection. -** If the db file can be opened for read-write access, it always is. Parameter -** bReadonly is only ever true if it has already been determined that the -** db can only be opened for read-only access. -** -** Return LSM_OK if successful or an lsm error code otherwise. -*/ -int lsmFsOpen( - lsm_db *pDb, /* Database connection to open fd for */ - const char *zDb, /* Full path to database file */ - int bReadonly /* True to open db file read-only */ -){ - FileSystem *pFS; - int rc = LSM_OK; - int nDb = strlen(zDb); - int nByte; - - assert( pDb->pFS==0 ); - assert( pDb->pWorker==0 && pDb->pClient==0 ); - - nByte = sizeof(FileSystem) + nDb+1 + nDb+4+1; - pFS = (FileSystem *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - if( pFS ){ - LsmFile *pLsmFile; - pFS->zDb = (char *)&pFS[1]; - pFS->zLog = &pFS->zDb[nDb+1]; - pFS->nPagesize = LSM_DFLT_PAGE_SIZE; - pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; - pFS->nMetasize = LSM_META_PAGE_SIZE; - pFS->nMetaRwSize = LSM_META_RW_PAGE_SIZE; - pFS->pDb = pDb; - pFS->pEnv = pDb->pEnv; - - /* Make a copy of the database and log file names. */ - memcpy(pFS->zDb, zDb, nDb+1); - memcpy(pFS->zLog, zDb, nDb); - memcpy(&pFS->zLog[nDb], "-log", 5); - - /* Allocate the hash-table here. At some point, it should be changed - ** so that it can grow dynamicly. */ - pFS->nCacheMax = 2048*1024 / pFS->nPagesize; - pFS->nHash = 4096; - pFS->apHash = lsmMallocZeroRc(pDb->pEnv, sizeof(Page *) * pFS->nHash, &rc); - - /* Open the database file */ - pLsmFile = lsmDbRecycleFd(pDb); - if( pLsmFile ){ - pFS->pLsmFile = pLsmFile; - pFS->fdDb = pLsmFile->pFile; - memset(pLsmFile, 0, sizeof(LsmFile)); - }else{ - pFS->pLsmFile = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmFile), &rc); - if( rc==LSM_OK ){ - pFS->fdDb = fsOpenFile(pFS, bReadonly, 0, &rc); - } - } - - if( rc!=LSM_OK ){ - lsmFsClose(pFS); - pFS = 0; - }else{ - pFS->szSector = lsmEnvSectorSize(pFS->pEnv, pFS->fdDb); - } - } - - pDb->pFS = pFS; - return rc; -} - -/* -** Configure the file-system object according to the current values of -** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. -*/ -int lsmFsConfigure(lsm_db *db){ - FileSystem *pFS = db->pFS; - if( pFS ){ - lsm_env *pEnv = pFS->pEnv; - Page *pPg; - - assert( pFS->nOut==0 ); - assert( pFS->pWaiting==0 ); - assert( pFS->pMapped==0 ); - - /* Reset any compression/decompression buffers already allocated */ - lsmFree(pEnv, pFS->aIBuffer); - lsmFree(pEnv, pFS->aOBuffer); - pFS->nBuffer = 0; - - /* Unmap the file, if it is currently mapped */ - if( pFS->pMap ){ - lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); - pFS->nMapLimit = 0; - } - - /* Free all allocated page structures */ - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - assert( pPg->flags & PAGE_FREE ); - lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - pPg = pFS->pFree; - while( pPg ){ - Page *pNext = pPg->pFreeNext; - lsmFree(pEnv, pPg); - pPg = pNext; - } - - /* Zero pointers that point to deleted page objects */ - pFS->nCacheAlloc = 0; - pFS->pLruFirst = 0; - pFS->pLruLast = 0; - pFS->pFree = 0; - if( pFS->apHash ){ - memset(pFS->apHash, 0, pFS->nHash*sizeof(pFS->apHash[0])); - } - - /* Configure the FileSystem object */ - if( db->compress.xCompress ){ - pFS->pCompress = &db->compress; - pFS->nMapLimit = 0; - }else{ - pFS->pCompress = 0; - if( db->iMmap==1 ){ - /* Unlimited */ - pFS->nMapLimit = (i64)1 << 60; - }else{ - /* iMmap is a limit in KB. Set nMapLimit to the same value in bytes. */ - pFS->nMapLimit = (i64)db->iMmap * 1024; - } - } - } - - return LSM_OK; -} - -/* -** Close and destroy a FileSystem object. -*/ -void lsmFsClose(FileSystem *pFS){ - if( pFS ){ - Page *pPg; - lsm_env *pEnv = pFS->pEnv; - - assert( pFS->nOut==0 ); - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - pPg = pFS->pFree; - while( pPg ){ - Page *pNext = pPg->pFreeNext; - if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - if( pFS->fdDb ) lsmEnvClose(pFS->pEnv, pFS->fdDb ); - if( pFS->fdLog ) lsmEnvClose(pFS->pEnv, pFS->fdLog ); - lsmFree(pEnv, pFS->pLsmFile); - lsmFree(pEnv, pFS->apHash); - lsmFree(pEnv, pFS->aIBuffer); - lsmFree(pEnv, pFS->aOBuffer); - lsmFree(pEnv, pFS); - } -} - -/* -** This function is called when closing a database handle (i.e. lsm_close()) -** if there exist other connections to the same database within this process. -** In that case the file-descriptor open on the database file is not closed -** when the FileSystem object is destroyed, as this would cause any POSIX -** locks held by the other connections to be silently dropped (see "man close" -** for details). Instead, the file-descriptor is stored in a list by the -** lsm_shared.c module until it is either closed or reused. -** -** This function returns a pointer to an object that can be linked into -** the list described above. The returned object now 'owns' the database -** file descriptor, so that when the FileSystem object is destroyed, it -** will not be closed. -** -** This function may be called at most once in the life-time of a -** FileSystem object. The results of any operations involving the database -** file descriptor are undefined once this function has been called. -** -** None of this is necessary on non-POSIX systems. But we do it anyway in -** the name of using as similar code as possible on all platforms. -*/ -LsmFile *lsmFsDeferClose(FileSystem *pFS){ - LsmFile *p = pFS->pLsmFile; - assert( p->pNext==0 ); - p->pFile = pFS->fdDb; - pFS->fdDb = 0; - pFS->pLsmFile = 0; - return p; -} - -/* -** Allocate a buffer and populate it with the output of the xFileid() -** method of the database file handle. If successful, set *ppId to point -** to the buffer and *pnId to the number of bytes in the buffer and return -** LSM_OK. Otherwise, set *ppId and *pnId to zero and return an LSM -** error code. -*/ -int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId){ - lsm_env *pEnv = pDb->pEnv; - FileSystem *pFS = pDb->pFS; - int rc; - int nId = 0; - void *pId; - - rc = pEnv->xFileid(pFS->fdDb, 0, &nId); - pId = lsmMallocZeroRc(pEnv, nId, &rc); - if( rc==LSM_OK ) rc = pEnv->xFileid(pFS->fdDb, pId, &nId); - - if( rc!=LSM_OK ){ - lsmFree(pEnv, pId); - pId = 0; - nId = 0; - } - - *ppId = pId; - *pnId = nId; - return rc; -} - -/* -** Return the nominal page-size used by this file-system. Actual pages -** may be smaller or larger than this value. -*/ -int lsmFsPageSize(FileSystem *pFS){ - return pFS->nPagesize; -} - -/* -** Return the block-size used by this file-system. -*/ -int lsmFsBlockSize(FileSystem *pFS){ - return pFS->nBlocksize; -} - -/* -** Configure the nominal page-size used by this file-system. Actual -** pages may be smaller or larger than this value. -*/ -void lsmFsSetPageSize(FileSystem *pFS, int nPgsz){ - pFS->nPagesize = nPgsz; - pFS->nCacheMax = 2048*1024 / pFS->nPagesize; -} - -/* -** Configure the block-size used by this file-system. -*/ -void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){ - pFS->nBlocksize = nBlocksize; -} - -/* -** Return the page number of the first page on block iBlock. Blocks are -** numbered starting from 1. -** -** For a compressed database, page numbers are byte offsets. The first -** page on each block is the byte offset immediately following the 4-byte -** "previous block" pointer at the start of each block. -*/ -static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){ - LsmPgno iPg; - if( pFS->pCompress ){ - if( iBlock==1 ){ - iPg = pFS->nMetasize * 2 + 4; - }else{ - iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4; - } - }else{ - const i64 nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - if( iBlock==1 ){ - iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize); - }else{ - iPg = 1 + (iBlock-1) * nPagePerBlock; - } - } - return iPg; -} - -/* -** Return the page number of the last page on block iBlock. Blocks are -** numbered starting from 1. -** -** For a compressed database, page numbers are byte offsets. The first -** page on each block is the byte offset of the byte immediately before -** the 4-byte "next block" pointer at the end of each block. -*/ -static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){ - if( pFS->pCompress ){ - return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4; - }else{ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - return iBlock * nPagePerBlock; - } -} - -/* -** Return the block number of the block that page iPg is located on. -** Blocks are numbered starting from 1. -*/ -static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){ - if( pFS->pCompress ){ - return (int)((iPg / pFS->nBlocksize) + 1); - }else{ - return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize))); - } -} - -/* -** Return true if page iPg is the last page on its block. -** -** This function is only called in non-compressed database mode. -*/ -static int fsIsLast(FileSystem *pFS, LsmPgno iPg){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - assert( !pFS->pCompress ); - return ( iPg && (iPg % nPagePerBlock)==0 ); -} - -/* -** Return true if page iPg is the first page on its block. -** -** This function is only called in non-compressed database mode. -*/ -static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - assert( !pFS->pCompress ); - return ( (iPg % nPagePerBlock)==1 - || (iPg<nPagePerBlock && iPg==fsFirstPageOnBlock(pFS, 1)) - ); -} - -/* -** Given a page reference, return a pointer to the buffer containing the -** pages contents. If parameter pnData is not NULL, set *pnData to the size -** of the buffer in bytes before returning. -*/ -u8 *lsmFsPageData(Page *pPage, int *pnData){ - if( pnData ){ - *pnData = pPage->nData; - } - return pPage->aData; -} - -/* -** Return the page number of a page. -*/ -LsmPgno lsmFsPageNumber(Page *pPage){ - /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */ - return pPage ? pPage->iPg : 0; -} - -/* -** Page pPg is currently part of the LRU list belonging to pFS. Remove -** it from the list. pPg->pLruNext and pPg->pLruPrev are cleared by this -** operation. -*/ -static void fsPageRemoveFromLru(FileSystem *pFS, Page *pPg){ - assert( pPg->pLruNext || pPg==pFS->pLruLast ); - assert( pPg->pLruPrev || pPg==pFS->pLruFirst ); - if( pPg->pLruNext ){ - pPg->pLruNext->pLruPrev = pPg->pLruPrev; - }else{ - pFS->pLruLast = pPg->pLruPrev; - } - if( pPg->pLruPrev ){ - pPg->pLruPrev->pLruNext = pPg->pLruNext; - }else{ - pFS->pLruFirst = pPg->pLruNext; - } - pPg->pLruPrev = 0; - pPg->pLruNext = 0; -} - -/* -** Page pPg is not currently part of the LRU list belonging to pFS. Add it. -*/ -static void fsPageAddToLru(FileSystem *pFS, Page *pPg){ - assert( pPg->pLruNext==0 && pPg->pLruPrev==0 ); - pPg->pLruPrev = pFS->pLruLast; - if( pPg->pLruPrev ){ - pPg->pLruPrev->pLruNext = pPg; - }else{ - pFS->pLruFirst = pPg; - } - pFS->pLruLast = pPg; -} - -/* -** Page pPg is currently stored in the apHash/nHash hash table. Remove it. -*/ -static void fsPageRemoveFromHash(FileSystem *pFS, Page *pPg){ - int iHash; - Page **pp; - - iHash = fsHashKey(pFS->nHash, pPg->iPg); - for(pp=&pFS->apHash[iHash]; *pp!=pPg; pp=&(*pp)->pHashNext); - *pp = pPg->pHashNext; - pPg->pHashNext = 0; -} - -/* -** Free a Page object allocated by fsPageBuffer(). -*/ -static void fsPageBufferFree(Page *pPg){ - pPg->pFS->nCacheAlloc--; - lsmFree(pPg->pFS->pEnv, pPg->aData); - lsmFree(pPg->pFS->pEnv, pPg); -} - - -/* -** Purge the cache of all non-mmap pages with nRef==0. -*/ -void lsmFsPurgeCache(FileSystem *pFS){ - Page *pPg; - - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - assert( pPg->flags & PAGE_FREE ); - fsPageRemoveFromHash(pFS, pPg); - fsPageBufferFree(pPg); - pPg = pNext; - } - pFS->pLruFirst = 0; - pFS->pLruLast = 0; - - assert( pFS->nCacheAlloc<=pFS->nOut && pFS->nCacheAlloc>=0 ); -} - -/* -** Search the hash-table for page iPg. If an entry is round, return a pointer -** to it. Otherwise, return NULL. -** -** Either way, if argument piHash is not NULL set *piHash to the hash slot -** number that page iPg would be stored in before returning. -*/ -static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){ - Page *p; /* Return value */ - int iHash = fsHashKey(pFS->nHash, iPg); - - if( piHash ) *piHash = iHash; - for(p=pFS->apHash[iHash]; p; p=p->pHashNext){ - if( p->iPg==iPg) break; - } - return p; -} - -/* -** Allocate and return a non-mmap Page object. If there are already -** nCacheMax such Page objects outstanding, try to recycle an existing -** Page instead. -*/ -static int fsPageBuffer( - FileSystem *pFS, - Page **ppOut -){ - int rc = LSM_OK; - Page *pPage = 0; - if( pFS->pLruFirst==0 || pFS->nCacheAlloc<pFS->nCacheMax ){ - /* Allocate a new Page object */ - pPage = lsmMallocZero(pFS->pEnv, sizeof(Page)); - if( !pPage ){ - rc = LSM_NOMEM_BKPT; - }else{ - pPage->aData = (u8 *)lsmMalloc(pFS->pEnv, pFS->nPagesize); - if( !pPage->aData ){ - lsmFree(pFS->pEnv, pPage); - rc = LSM_NOMEM_BKPT; - pPage = 0; - }else{ - pFS->nCacheAlloc++; - } - } - }else{ - /* Reuse an existing Page object */ - u8 *aData; - pPage = pFS->pLruFirst; - aData = pPage->aData; - fsPageRemoveFromLru(pFS, pPage); - fsPageRemoveFromHash(pFS, pPage); - - memset(pPage, 0, sizeof(Page)); - pPage->aData = aData; - } - - if( pPage ){ - pPage->flags = PAGE_FREE; - } - *ppOut = pPage; - return rc; -} - -/* -** Assuming *pRc is initially LSM_OK, attempt to ensure that the -** memory-mapped region is at least iSz bytes in size. If it is not already, -** iSz bytes in size, extend it and update the pointers associated with any -** outstanding Page objects. -** -** If *pRc is not LSM_OK when this function is called, it is a no-op. -** Otherwise, *pRc is set to an lsm error code if an error occurs, or -** left unmodified otherwise. -** -** This function is never called in compressed database mode. -*/ -static void fsGrowMapping( - FileSystem *pFS, /* File system object */ - i64 iSz, /* Minimum size to extend mapping to */ - int *pRc /* IN/OUT: Error code */ -){ - assert( PAGE_HASPREV==4 ); - - if( *pRc==LSM_OK && iSz>pFS->nMap ){ - int rc; - u8 *aOld = pFS->pMap; - rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, iSz, &pFS->pMap, &pFS->nMap); - if( rc==LSM_OK && pFS->pMap!=aOld ){ - Page *pFix; - i64 iOff = (u8 *)pFS->pMap - aOld; - for(pFix=pFS->pMapped; pFix; pFix=pFix->pMappedNext){ - pFix->aData += iOff; - } - lsmSortedRemap(pFS->pDb); - } - *pRc = rc; - } -} - -/* -** If it is mapped, unmap the database file. -*/ -int lsmFsUnmap(FileSystem *pFS){ - int rc = LSM_OK; - if( pFS ){ - rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); - } - return rc; -} - -/* -** fsync() the database file. -*/ -int lsmFsSyncDb(FileSystem *pFS, int nBlock){ - return lsmEnvSync(pFS->pEnv, pFS->fdDb); -} - -/* -** If block iBlk has been redirected according to the redirections in the -** object passed as the first argument, return the destination block to -** which it is redirected. Otherwise, return a copy of iBlk. -*/ -static int fsRedirectBlock(Redirect *p, int iBlk){ - if( p ){ - int i; - for(i=0; i<p->n; i++){ - if( iBlk==p->a[i].iFrom ) return p->a[i].iTo; - } - } - assert( iBlk!=0 ); - return iBlk; -} - -/* -** If page iPg has been redirected according to the redirections in the -** object passed as the second argument, return the destination page to -** which it is redirected. Otherwise, return a copy of iPg. -*/ -LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){ - LsmPgno iReal = iPg; - - if( pRedir ){ - const int nPagePerBlock = ( - pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) - ); - int iBlk = fsPageToBlock(pFS, iPg); - int i; - for(i=0; i<pRedir->n; i++){ - int iFrom = pRedir->a[i].iFrom; - if( iFrom>iBlk ) break; - if( iFrom==iBlk ){ - int iTo = pRedir->a[i].iTo; - iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; - if( iTo==1 ){ - iReal += (fsFirstPageOnBlock(pFS, 1)-1); - } - break; - } - } - } - - assert( iReal!=0 ); - return iReal; -} - -/* Required by the circular fsBlockNext<->fsPageGet dependency. */ -static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *); - -/* -** Parameter iBlock is a database file block. This function reads the value -** stored in the blocks "next block" pointer and stores it in *piNext. -** LSM_OK is returned if everything is successful, or an LSM error code -** otherwise. -*/ -static int fsBlockNext( - FileSystem *pFS, /* File-system object handle */ - Segment *pSeg, /* Use this segment for block redirects */ - int iBlock, /* Read field from this block */ - int *piNext /* OUT: Next block in linked list */ -){ - int rc; - int iRead; /* Read block from here */ - - if( pSeg ){ - iRead = fsRedirectBlock(pSeg->pRedirect, iBlock); - }else{ - iRead = iBlock; - } - - assert( pFS->nMapLimit==0 || pFS->pCompress==0 ); - if( pFS->pCompress ){ - i64 iOff; /* File offset to read data from */ - u8 aNext[4]; /* 4-byte pointer read from db file */ - - iOff = (i64)iRead * pFS->nBlocksize - sizeof(aNext); - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aNext, sizeof(aNext)); - if( rc==LSM_OK ){ - *piNext = (int)lsmGetU32(aNext); - } - }else{ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - Page *pLast; - rc = fsPageGet(pFS, 0, iRead*nPagePerBlock, 0, &pLast, 0); - if( rc==LSM_OK ){ - *piNext = lsmGetU32(&pLast->aData[pFS->nPagesize-4]); - lsmFsPageRelease(pLast); - } - } - - if( pSeg ){ - *piNext = fsRedirectBlock(pSeg->pRedirect, *piNext); - } - return rc; -} - -/* -** Return the page number of the last page on the same block as page iPg. -*/ -LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){ - return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg)); -} - -/* -** Read nData bytes of data from offset iOff of the database file into -** buffer aData. If this means reading past the end of a block, follow -** the block pointer to the next block and continue reading. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** This function is only called in compressed database mode. -*/ -static int fsReadData( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Block redirection */ - i64 iOff, /* Read data from this offset */ - u8 *aData, /* Buffer to read data into */ - int nData /* Number of bytes to read */ -){ - i64 iEob; /* End of block */ - int nRead; - int rc; - - assert( pFS->pCompress ); - - iEob = fsLastPageOnPagesBlock(pFS, iOff) + 1; - nRead = (int)LSM_MIN(iEob - iOff, nData); - - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nRead); - if( rc==LSM_OK && nRead!=nData ){ - int iBlk; - - rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - if( rc==LSM_OK ){ - i64 iOff2 = fsFirstPageOnBlock(pFS, iBlk); - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff2, &aData[nRead], nData-nRead); - } - } - - return rc; -} - -/* -** Parameter iBlock is a database file block. This function reads the value -** stored in the blocks "previous block" pointer and stores it in *piPrev. -** LSM_OK is returned if everything is successful, or an LSM error code -** otherwise. -*/ -static int fsBlockPrev( - FileSystem *pFS, /* File-system object handle */ - Segment *pSeg, /* Use this segment for block redirects */ - int iBlock, /* Read field from this block */ - int *piPrev /* OUT: Previous block in linked list */ -){ - int rc = LSM_OK; /* Return code */ - - assert( pFS->nMapLimit==0 || pFS->pCompress==0 ); - assert( iBlock>0 ); - - if( pFS->pCompress ){ - i64 iOff = fsFirstPageOnBlock(pFS, iBlock) - 4; - u8 aPrev[4]; /* 4-byte pointer read from db file */ - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aPrev, sizeof(aPrev)); - if( rc==LSM_OK ){ - Redirect *pRedir = (pSeg ? pSeg->pRedirect : 0); - *piPrev = fsRedirectBlock(pRedir, (int)lsmGetU32(aPrev)); - } - }else{ - assert( 0 ); - } - return rc; -} - -/* -** Encode and decode routines for record size fields. -*/ -static void putRecordSize(u8 *aBuf, int nByte, int bFree){ - aBuf[0] = (u8)(nByte >> 14) | 0x80; - aBuf[1] = ((u8)(nByte >> 7) & 0x7F) | (bFree ? 0x00 : 0x80); - aBuf[2] = (u8)nByte | 0x80; -} -static int getRecordSize(u8 *aBuf, int *pbFree){ - int nByte; - nByte = (aBuf[0] & 0x7F) << 14; - nByte += (aBuf[1] & 0x7F) << 7; - nByte += (aBuf[2] & 0x7F); - *pbFree = !(aBuf[1] & 0x80); - return nByte; -} - -/* -** Subtract iSub from database file offset iOff and set *piRes to the -** result. If doing so means passing the start of a block, follow the -** block pointer stored in the first 4 bytes of the block. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -static int fsSubtractOffset( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - int iSub, - i64 *piRes -){ - i64 iStart; - int iBlk = 0; - int rc; - - assert( pFS->pCompress ); - - iStart = fsFirstPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); - if( (iOff-iSub)>=iStart ){ - *piRes = (iOff-iSub); - return LSM_OK; - } - - rc = fsBlockPrev(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - *piRes = fsLastPageOnBlock(pFS, iBlk) - iSub + (iOff - iStart + 1); - return rc; -} - -/* -** Add iAdd to database file offset iOff and set *piRes to the -** result. If doing so means passing the end of a block, follow the -** block pointer stored in the last 4 bytes of the block. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -static int fsAddOffset( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - int iAdd, - i64 *piRes -){ - i64 iEob; - int iBlk; - int rc; - - assert( pFS->pCompress ); - - iEob = fsLastPageOnPagesBlock(pFS, iOff); - if( (iOff+iAdd)<=iEob ){ - *piRes = (iOff+iAdd); - return LSM_OK; - } - - rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1); - return rc; -} - -/* -** If it is not already allocated, allocate either the FileSystem.aOBuffer (if -** bWrite is true) or the FileSystem.aIBuffer (if bWrite is false). Return -** LSM_OK if successful if the attempt to allocate memory fails. -*/ -static int fsAllocateBuffer(FileSystem *pFS, int bWrite){ - u8 **pp; /* Pointer to either aIBuffer or aOBuffer */ - - assert( pFS->pCompress ); - - /* If neither buffer has been allocated, figure out how large they - ** should be. Store this value in FileSystem.nBuffer. */ - if( pFS->nBuffer==0 ){ - assert( pFS->aIBuffer==0 && pFS->aOBuffer==0 ); - pFS->nBuffer = pFS->pCompress->xBound(pFS->pCompress->pCtx, pFS->nPagesize); - if( pFS->nBuffer<(pFS->szSector+6) ){ - pFS->nBuffer = pFS->szSector+6; - } - } - - pp = (bWrite ? &pFS->aOBuffer : &pFS->aIBuffer); - if( *pp==0 ){ - *pp = lsmMalloc(pFS->pEnv, LSM_MAX(pFS->nBuffer, pFS->nPagesize)); - if( *pp==0 ) return LSM_NOMEM_BKPT; - } - - return LSM_OK; -} - -/* -** This function is only called in compressed database mode. It reads and -** uncompresses the compressed data for page pPg from the database and -** populates the pPg->aData[] buffer and pPg->nCompress field. -** -** It is possible that instead of a page record, there is free space -** at offset pPg->iPgno. In this case no data is read from the file, but -** output variable *pnSpace is set to the total number of free bytes. -** -** LSM_OK is returned if successful, or an LSM error code otherwise. -*/ -static int fsReadPagedata( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* pPg is part of this segment */ - Page *pPg, /* Page to read and uncompress data for */ - int *pnSpace /* OUT: Total bytes of free space */ -){ - lsm_compress *p = pFS->pCompress; - i64 iOff = pPg->iPg; - u8 aSz[3]; - int rc; - - assert( p && pPg->nCompress==0 ); - - if( fsAllocateBuffer(pFS, 0) ) return LSM_NOMEM; - - rc = fsReadData(pFS, pSeg, iOff, aSz, sizeof(aSz)); - - if( rc==LSM_OK ){ - int bFree; - if( aSz[0] & 0x80 ){ - pPg->nCompress = (int)getRecordSize(aSz, &bFree); - }else{ - pPg->nCompress = (int)aSz[0] - sizeof(aSz)*2; - bFree = 1; - } - if( bFree ){ - if( pnSpace ){ - *pnSpace = pPg->nCompress + sizeof(aSz)*2; - }else{ - rc = LSM_CORRUPT_BKPT; - } - }else{ - rc = fsAddOffset(pFS, pSeg, iOff, 3, &iOff); - if( rc==LSM_OK ){ - if( pPg->nCompress>pFS->nBuffer ){ - rc = LSM_CORRUPT_BKPT; - }else{ - rc = fsReadData(pFS, pSeg, iOff, pFS->aIBuffer, pPg->nCompress); - } - if( rc==LSM_OK ){ - int n = pFS->nPagesize; - rc = p->xUncompress(p->pCtx, - (char *)pPg->aData, &n, - (const char *)pFS->aIBuffer, pPg->nCompress - ); - if( rc==LSM_OK && n!=pPg->pFS->nPagesize ){ - rc = LSM_CORRUPT_BKPT; - } - } - } - } - } - return rc; -} - -/* -** Return a handle for a database page. -** -** If this file-system object is accessing a compressed database it may be -** that there is no page record at database file offset iPg. Instead, there -** may be a free space record. In this case, set *ppPg to NULL and *pnSpace -** to the total number of free bytes before returning. -** -** If no error occurs, LSM_OK is returned. Otherwise, an lsm error code. -*/ -static int fsPageGet( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Block redirection to use (or NULL) */ - LsmPgno iPg, /* Page id */ - int noContent, /* True to not load content from disk */ - Page **ppPg, /* OUT: New page handle */ - int *pnSpace /* OUT: Bytes of free space */ -){ - Page *p; - int iHash; - int rc = LSM_OK; - - /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is - ** not NULL, and the block containing iPg has been redirected, then iReal - ** is the page number after redirection. */ - LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg); - - assert_lists_are_ok(pFS); - assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); - assert( iReal>=fsFirstPageOnBlock(pFS, 1) ); - *ppPg = 0; - - /* Search the hash-table for the page */ - p = fsPageFindInHash(pFS, iReal, &iHash); - - if( p ){ - assert( p->flags & PAGE_FREE ); - if( p->nRef==0 ) fsPageRemoveFromLru(pFS, p); - }else{ - - if( fsMmapPage(pFS, iReal) ){ - i64 iEnd = (i64)iReal * pFS->nPagesize; - fsGrowMapping(pFS, iEnd, &rc); - if( rc!=LSM_OK ) return rc; - - if( pFS->pFree ){ - p = pFS->pFree; - pFS->pFree = p->pFreeNext; - assert( p->nRef==0 ); - }else{ - p = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); - if( rc ) return rc; - p->pFS = pFS; - } - p->aData = &((u8 *)pFS->pMap)[pFS->nPagesize * (iReal-1)]; - p->iPg = iReal; - - /* This page now carries a pointer to the mapping. Link it in to - ** the FileSystem.pMapped list. */ - assert( p->pMappedNext==0 ); - p->pMappedNext = pFS->pMapped; - pFS->pMapped = p; - - assert( pFS->pCompress==0 ); - assert( (p->flags & PAGE_FREE)==0 ); - }else{ - rc = fsPageBuffer(pFS, &p); - if( rc==LSM_OK ){ - int nSpace = 0; - p->iPg = iReal; - p->nRef = 0; - p->pFS = pFS; - assert( p->flags==0 || p->flags==PAGE_FREE ); - -#ifdef LSM_DEBUG - memset(p->aData, 0x56, pFS->nPagesize); -#endif - assert( p->pLruNext==0 && p->pLruPrev==0 ); - if( noContent==0 ){ - if( pFS->pCompress ){ - rc = fsReadPagedata(pFS, pSeg, p, &nSpace); - }else{ - int nByte = pFS->nPagesize; - i64 iOff = (i64)(iReal-1) * pFS->nPagesize; - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, p->aData, nByte); - } - pFS->nRead++; - } - - /* If the xRead() call was successful (or not attempted), link the - ** page into the page-cache hash-table. Otherwise, if it failed, - ** free the buffer. */ - if( rc==LSM_OK && nSpace==0 ){ - p->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = p; - }else{ - fsPageBufferFree(p); - p = 0; - if( pnSpace ) *pnSpace = nSpace; - } - } - } - - assert( (rc==LSM_OK && (p || (pnSpace && *pnSpace))) - || (rc!=LSM_OK && p==0) - ); - } - - if( rc==LSM_OK && p ){ - if( pFS->pCompress==0 && (fsIsLast(pFS, iReal) || fsIsFirst(pFS, iReal)) ){ - p->nData = pFS->nPagesize - 4; - if( fsIsFirst(pFS, iReal) && p->nRef==0 ){ - p->aData += 4; - p->flags |= PAGE_HASPREV; - } - }else{ - p->nData = pFS->nPagesize; - } - pFS->nOut += (p->nRef==0); - p->nRef++; - } - *ppPg = p; - return rc; -} - -/* -** Read the 64-bit checkpoint id of the checkpoint currently stored on meta -** page iMeta of the database file. If no error occurs, store the id value -** in *piVal and return LSM_OK. Otherwise, return an LSM error code and leave -** *piVal unmodified. -** -** If a checkpointer connection is currently updating meta-page iMeta, or an -** earlier checkpointer crashed while doing so, the value read into *piVal -** may be garbage. It is the callers responsibility to deal with this. -*/ -int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){ - FileSystem *pFS = db->pFS; - int rc = LSM_OK; - - assert( iMeta==1 || iMeta==2 ); - if( pFS->nMapLimit>0 ){ - fsGrowMapping(pFS, iMeta*LSM_META_PAGE_SIZE, &rc); - if( rc==LSM_OK ){ - *piVal = (i64)lsmGetU64(&((u8 *)pFS->pMap)[(iMeta-1)*LSM_META_PAGE_SIZE]); - } - }else{ - MetaPage *pMeta = 0; - rc = lsmFsMetaPageGet(pFS, 0, iMeta, &pMeta); - if( rc==LSM_OK ){ - *piVal = (i64)lsmGetU64(pMeta->aData); - lsmFsMetaPageRelease(pMeta); - } - } - - return rc; -} - - -/* -** Return true if the first or last page of segment pRun falls between iFirst -** and iLast, inclusive, and pRun is not equal to pIgnore. -*/ -static int fsRunEndsBetween( - Segment *pRun, - Segment *pIgnore, - LsmPgno iFirst, - LsmPgno iLast -){ - return (pRun!=pIgnore && ( - (pRun->iFirst>=iFirst && pRun->iFirst<=iLast) - || (pRun->iLastPg>=iFirst && pRun->iLastPg<=iLast) - )); -} - -/* -** Return true if level pLevel contains a segment other than pIgnore for -** which the first or last page is between iFirst and iLast, inclusive. -*/ -static int fsLevelEndsBetween( - Level *pLevel, - Segment *pIgnore, - LsmPgno iFirst, - LsmPgno iLast -){ - int i; - - if( fsRunEndsBetween(&pLevel->lhs, pIgnore, iFirst, iLast) ){ - return 1; - } - for(i=0; i<pLevel->nRight; i++){ - if( fsRunEndsBetween(&pLevel->aRhs[i], pIgnore, iFirst, iLast) ){ - return 1; - } - } - - return 0; -} - -/* -** Block iBlk is no longer in use by segment pIgnore. If it is not in use -** by any other segment, move it to the free block list. -*/ -static int fsFreeBlock( - FileSystem *pFS, /* File system object */ - Snapshot *pSnapshot, /* Worker snapshot */ - Segment *pIgnore, /* Ignore this run when searching */ - int iBlk /* Block number of block to free */ -){ - int rc = LSM_OK; /* Return code */ - LsmPgno iFirst; /* First page on block iBlk */ - LsmPgno iLast; /* Last page on block iBlk */ - Level *pLevel; /* Used to iterate through levels */ - - int iIn; /* Used to iterate through append points */ - int iOut = 0; /* Used to output append points */ - LsmPgno *aApp = pSnapshot->aiAppend; - - iFirst = fsFirstPageOnBlock(pFS, iBlk); - iLast = fsLastPageOnBlock(pFS, iBlk); - - /* Check if any other run in the snapshot has a start or end page - ** within this block. If there is such a run, return early. */ - for(pLevel=lsmDbSnapshotLevel(pSnapshot); pLevel; pLevel=pLevel->pNext){ - if( fsLevelEndsBetween(pLevel, pIgnore, iFirst, iLast) ){ - return LSM_OK; - } - } - - /* Remove any entries that lie on this block from the append-list. */ - for(iIn=0; iIn<LSM_APPLIST_SZ; iIn++){ - if( aApp[iIn]<iFirst || aApp[iIn]>iLast ){ - aApp[iOut++] = aApp[iIn]; - } - } - while( iOut<LSM_APPLIST_SZ ) aApp[iOut++] = 0; - - if( rc==LSM_OK ){ - rc = lsmBlockFree(pFS->pDb, iBlk); - } - return rc; -} - -/* -** Delete or otherwise recycle the blocks currently occupied by run pDel. -*/ -int lsmFsSortedDelete( - FileSystem *pFS, - Snapshot *pSnapshot, - int bZero, /* True to zero the Segment structure */ - Segment *pDel -){ - if( pDel->iFirst ){ - int rc = LSM_OK; - - int iBlk; - int iLastBlk; - - iBlk = fsPageToBlock(pFS, pDel->iFirst); - iLastBlk = fsPageToBlock(pFS, pDel->iLastPg); - - /* Mark all blocks currently used by this sorted run as free */ - while( iBlk && rc==LSM_OK ){ - int iNext = 0; - if( iBlk!=iLastBlk ){ - rc = fsBlockNext(pFS, pDel, iBlk, &iNext); - }else if( bZero==0 && pDel->iLastPg!=fsLastPageOnBlock(pFS, iLastBlk) ){ - break; - } - rc = fsFreeBlock(pFS, pSnapshot, pDel, iBlk); - iBlk = iNext; - } - - if( pDel->pRedirect ){ - assert( pDel->pRedirect==&pSnapshot->redirect ); - pSnapshot->redirect.n = 0; - } - - if( bZero ) memset(pDel, 0, sizeof(Segment)); - } - return LSM_OK; -} - -/* -** aPgno is an array containing nPgno page numbers. Return the smallest page -** number from the array that falls on block iBlk. Or, if none of the pages -** in aPgno[] fall on block iBlk, return 0. -*/ -static LsmPgno firstOnBlock( - FileSystem *pFS, - int iBlk, - LsmPgno *aPgno, - int nPgno -){ - LsmPgno iRet = 0; - int i; - for(i=0; i<nPgno; i++){ - LsmPgno iPg = aPgno[i]; - if( fsPageToBlock(pFS, iPg)==iBlk && (iRet==0 || iPg<iRet) ){ - iRet = iPg; - } - } - return iRet; -} - -#ifndef NDEBUG -/* -** Return true if page iPg, which is a part of segment p, lies on -** a redirected block. -*/ -static int fsPageRedirects(FileSystem *pFS, Segment *p, LsmPgno iPg){ - return (iPg!=0 && iPg!=lsmFsRedirectPage(pFS, p->pRedirect, iPg)); -} - -/* -** Return true if the second argument is not NULL and any of the first -** last or root pages lie on a redirected block. -*/ -static int fsSegmentRedirects(FileSystem *pFS, Segment *p){ - return (p && ( - fsPageRedirects(pFS, p, p->iFirst) - || fsPageRedirects(pFS, p, p->iRoot) - || fsPageRedirects(pFS, p, p->iLastPg) - )); -} -#endif - -/* -** Argument aPgno is an array of nPgno page numbers. All pages belong to -** the segment pRun. This function gobbles from the start of the run to the -** first page that appears in aPgno[] (i.e. so that the aPgno[] entry is -** the new first page of the run). -*/ -void lsmFsGobble( - lsm_db *pDb, - Segment *pRun, - LsmPgno *aPgno, - int nPgno -){ - int rc = LSM_OK; - FileSystem *pFS = pDb->pFS; - Snapshot *pSnapshot = pDb->pWorker; - int iBlk; - - assert( pRun->nSize>0 ); - assert( 0==fsSegmentRedirects(pFS, pRun) ); - assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) ); - - iBlk = fsPageToBlock(pFS, pRun->iFirst); - pRun->nSize += (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); - - while( rc==LSM_OK ){ - int iNext = 0; - LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno); - if( iFirst ){ - pRun->iFirst = iFirst; - break; - } - rc = fsBlockNext(pFS, pRun, iBlk, &iNext); - if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk); - pRun->nSize -= ( - 1 + fsLastPageOnBlock(pFS, iBlk) - fsFirstPageOnBlock(pFS, iBlk) - ); - iBlk = iNext; - } - - pRun->nSize -= (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); - assert( pRun->nSize>0 ); -} - -/* -** This function is only used in compressed database mode. -** -** Argument iPg is the page number (byte offset) of a page within segment -** pSeg. The page record, including all headers, is nByte bytes in size. -** Before returning, set *piNext to the page number of the next page in -** the segment, or to zero if iPg is the last. -** -** In other words, do: -** -** *piNext = iPg + nByte; -** -** But take block overflow and redirection into account. -*/ -static int fsNextPageOffset( - FileSystem *pFS, /* File system object */ - Segment *pSeg, /* Segment to move within */ - LsmPgno iPg, /* Offset of current page */ - int nByte, /* Size of current page including headers */ - LsmPgno *piNext /* OUT: Offset of next page. Or zero (EOF) */ -){ - LsmPgno iNext; - int rc; - - assert( pFS->pCompress ); - - rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext); - if( pSeg && iNext==pSeg->iLastPg ){ - iNext = 0; - }else if( rc==LSM_OK ){ - rc = fsAddOffset(pFS, pSeg, iNext, 1, &iNext); - } - - *piNext = iNext; - return rc; -} - -/* -** This function is only used in compressed database mode. -** -** Argument iPg is the page number of a pagethat appears in segment pSeg. -** This function determines the page number of the previous page in the -** same run. *piPrev is set to the previous page number before returning. -** -** LSM_OK is returned if no error occurs. Otherwise, an lsm error code. -** If any value other than LSM_OK is returned, then the final value of -** *piPrev is undefined. -*/ -static int fsGetPageBefore( - FileSystem *pFS, - Segment *pSeg, - LsmPgno iPg, - LsmPgno *piPrev -){ - u8 aSz[3]; - int rc; - i64 iRead; - - assert( pFS->pCompress ); - - rc = fsSubtractOffset(pFS, pSeg, iPg, sizeof(aSz), &iRead); - if( rc==LSM_OK ) rc = fsReadData(pFS, pSeg, iRead, aSz, sizeof(aSz)); - - if( rc==LSM_OK ){ - int bFree; - int nSz; - if( aSz[2] & 0x80 ){ - nSz = getRecordSize(aSz, &bFree) + sizeof(aSz)*2; - }else{ - nSz = (int)(aSz[2] & 0x7F); - bFree = 1; - } - rc = fsSubtractOffset(pFS, pSeg, iPg, nSz, piPrev); - } - - return rc; -} - -/* -** The first argument to this function is a valid reference to a database -** file page that is part of a sorted run. If parameter eDir is -1, this -** function attempts to locate and load the previous page in the same run. -** Or, if eDir is +1, it attempts to find the next page in the same run. -** The results of passing an eDir value other than positive or negative one -** are undefined. -** -** If parameter pRun is not NULL then it must point to the run that page -** pPg belongs to. In this case, if pPg is the first or last page of the -** run, and the request is for the previous or next page, respectively, -** *ppNext is set to NULL before returning LSM_OK. If pRun is NULL, then it -** is assumed that the next or previous page, as requested, exists. -** -** If the previous/next page does exist and is successfully loaded, *ppNext -** is set to point to it and LSM_OK is returned. Otherwise, if an error -** occurs, *ppNext is set to NULL and and lsm error code returned. -** -** Page references returned by this function should be released by the -** caller using lsmFsPageRelease(). -*/ -int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){ - int rc = LSM_OK; - FileSystem *pFS = pPg->pFS; - LsmPgno iPg = pPg->iPg; - - assert( 0==fsSegmentRedirects(pFS, pRun) ); - if( pFS->pCompress ){ - int nSpace = pPg->nCompress + 2*3; - - do { - if( eDir>0 ){ - rc = fsNextPageOffset(pFS, pRun, iPg, nSpace, &iPg); - }else{ - if( iPg==pRun->iFirst ){ - iPg = 0; - }else{ - rc = fsGetPageBefore(pFS, pRun, iPg, &iPg); - } - } - - nSpace = 0; - if( iPg!=0 ){ - rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, &nSpace); - assert( (*ppNext==0)==(rc!=LSM_OK || nSpace>0) ); - }else{ - *ppNext = 0; - } - }while( nSpace>0 && rc==LSM_OK ); - - }else{ - Redirect *pRedir = pRun ? pRun->pRedirect : 0; - assert( eDir==1 || eDir==-1 ); - if( eDir<0 ){ - if( pRun && iPg==pRun->iFirst ){ - *ppNext = 0; - return LSM_OK; - }else if( fsIsFirst(pFS, iPg) ){ - assert( pPg->flags & PAGE_HASPREV ); - iPg = fsLastPageOnBlock(pFS, lsmGetU32(&pPg->aData[-4])); - }else{ - iPg--; - } - }else{ - if( pRun ){ - if( iPg==pRun->iLastPg ){ - *ppNext = 0; - return LSM_OK; - } - } - - if( fsIsLast(pFS, iPg) ){ - int iBlk = fsRedirectBlock( - pRedir, lsmGetU32(&pPg->aData[pFS->nPagesize-4]) - ); - iPg = fsFirstPageOnBlock(pFS, iBlk); - }else{ - iPg++; - } - } - rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, 0); - } - - return rc; -} - -/* -** This function is called when creating a new segment to determine if the -** first part of it can be written following an existing segment on an -** already allocated block. If it is possible, the page number of the first -** page to use for the new segment is returned. Otherwise zero. -** -** If argument pLvl is not NULL, then this function will not attempt to -** start the new segment immediately following any segment that is part -** of the right-hand-side of pLvl. -*/ -static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){ - int i; - LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; - LsmPgno iRet = 0; - - for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){ - if( (iRet = aiAppend[i]) ){ - if( pLvl ){ - int iBlk = fsPageToBlock(pFS, iRet); - int j; - for(j=0; iRet && j<pLvl->nRight; j++){ - if( fsPageToBlock(pFS, pLvl->aRhs[j].iLastPg)==iBlk ){ - iRet = 0; - } - } - } - if( iRet ) aiAppend[i] = 0; - } - } - return iRet; -} - -/* -** Append a page to the left-hand-side of pLvl. Set the ref-count to 1 and -** return a pointer to it. The page is writable until either -** lsmFsPagePersist() is called on it or the ref-count drops to zero. -*/ -int lsmFsSortedAppend( - FileSystem *pFS, - Snapshot *pSnapshot, - Level *pLvl, - int bDefer, - Page **ppOut -){ - int rc = LSM_OK; - Page *pPg = 0; - LsmPgno iApp = 0; - LsmPgno iNext = 0; - Segment *p = &pLvl->lhs; - LsmPgno iPrev = p->iLastPg; - - *ppOut = 0; - assert( p->pRedirect==0 ); - - if( pFS->pCompress || bDefer ){ - /* In compressed database mode the page is not assigned a page number - ** or location in the database file at this point. This will be done - ** by the lsmFsPagePersist() call. */ - rc = fsPageBuffer(pFS, &pPg); - if( rc==LSM_OK ){ - pPg->pFS = pFS; - pPg->pSeg = p; - pPg->iPg = 0; - pPg->flags |= PAGE_DIRTY; - pPg->nData = pFS->nPagesize; - assert( pPg->aData ); - if( pFS->pCompress==0 ) pPg->nData -= 4; - - pPg->nRef = 1; - pFS->nOut++; - } - }else{ - if( iPrev==0 ){ - iApp = findAppendPoint(pFS, pLvl); - }else if( fsIsLast(pFS, iPrev) ){ - int iNext2; - rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iPrev), &iNext2); - if( rc!=LSM_OK ) return rc; - iApp = fsFirstPageOnBlock(pFS, iNext2); - }else{ - iApp = iPrev + 1; - } - - /* If this is the first page allocated, or if the page allocated is the - ** last in the block, also allocate the next block here. */ - if( iApp==0 || fsIsLast(pFS, iApp) ){ - int iNew; /* New block number */ - - rc = lsmBlockAllocate(pFS->pDb, 0, &iNew); - if( rc!=LSM_OK ) return rc; - if( iApp==0 ){ - iApp = fsFirstPageOnBlock(pFS, iNew); - }else{ - iNext = fsFirstPageOnBlock(pFS, iNew); - } - } - - /* Grab the new page. */ - pPg = 0; - rc = fsPageGet(pFS, 0, iApp, 1, &pPg, 0); - assert( rc==LSM_OK || pPg==0 ); - - /* If this is the first or last page of a block, fill in the pointer - ** value at the end of the new page. */ - if( rc==LSM_OK ){ - p->nSize++; - p->iLastPg = iApp; - if( p->iFirst==0 ) p->iFirst = iApp; - pPg->flags |= PAGE_DIRTY; - - if( fsIsLast(pFS, iApp) ){ - lsmPutU32(&pPg->aData[pFS->nPagesize-4], fsPageToBlock(pFS, iNext)); - }else if( fsIsFirst(pFS, iApp) ){ - lsmPutU32(&pPg->aData[-4], fsPageToBlock(pFS, iPrev)); - } - } - } - - *ppOut = pPg; - return rc; -} - -/* -** Mark the segment passed as the second argument as finished. Once a segment -** is marked as finished it is not possible to append any further pages to -** it. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -int lsmFsSortedFinish(FileSystem *pFS, Segment *p){ - int rc = LSM_OK; - if( p && p->iLastPg ){ - assert( p->pRedirect==0 ); - - /* Check if the last page of this run happens to be the last of a block. - ** If it is, then an extra block has already been allocated for this run. - ** Shift this extra block back to the free-block list. - ** - ** Otherwise, add the first free page in the last block used by the run - ** to the lAppend list. - */ - if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){ - int i; - LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; - for(i=0; i<LSM_APPLIST_SZ; i++){ - if( aiAppend[i]==0 ){ - aiAppend[i] = p->iLastPg+1; - break; - } - } - }else if( pFS->pCompress==0 ){ - Page *pLast; - rc = fsPageGet(pFS, 0, p->iLastPg, 0, &pLast, 0); - if( rc==LSM_OK ){ - int iBlk = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]); - lsmBlockRefree(pFS->pDb, iBlk); - lsmFsPageRelease(pLast); - } - }else{ - int iBlk = 0; - rc = fsBlockNext(pFS, p, fsPageToBlock(pFS, p->iLastPg), &iBlk); - if( rc==LSM_OK ){ - lsmBlockRefree(pFS->pDb, iBlk); - } - } - } - return rc; -} - -/* -** Obtain a reference to page number iPg. -** -** Return LSM_OK if successful, or an lsm error code if an error occurs. -*/ -int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){ - return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); -} - -/* -** Obtain a reference to the last page in the segment passed as the -** second argument. -** -** Return LSM_OK if successful, or an lsm error code if an error occurs. -*/ -int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){ - int rc; - LsmPgno iPg = pSeg->iLastPg; - if( pFS->pCompress ){ - int nSpace; - iPg++; - do { - nSpace = 0; - rc = fsGetPageBefore(pFS, pSeg, iPg, &iPg); - if( rc==LSM_OK ){ - rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, &nSpace); - } - }while( rc==LSM_OK && nSpace>0 ); - - }else{ - rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); - } - return rc; -} - -/* -** Return a reference to meta-page iPg. If successful, LSM_OK is returned -** and *ppPg populated with the new page reference. The reference should -** be released by the caller using lsmFsPageRelease(). -** -** Otherwise, if an error occurs, *ppPg is set to NULL and an LSM error -** code is returned. -*/ -int lsmFsMetaPageGet( - FileSystem *pFS, /* File-system connection */ - int bWrite, /* True for write access, false for read */ - int iPg, /* Either 1 or 2 */ - MetaPage **ppPg /* OUT: Pointer to MetaPage object */ -){ - int rc = LSM_OK; - MetaPage *pPg; - assert( iPg==1 || iPg==2 ); - - pPg = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); - - if( pPg ){ - i64 iOff = (iPg-1) * pFS->nMetasize; - if( pFS->nMapLimit>0 ){ - fsGrowMapping(pFS, 2*pFS->nMetasize, &rc); - pPg->aData = (u8 *)(pFS->pMap) + iOff; - }else{ - pPg->aData = lsmMallocRc(pFS->pEnv, pFS->nMetasize, &rc); - if( rc==LSM_OK && bWrite==0 ){ - rc = lsmEnvRead( - pFS->pEnv, pFS->fdDb, iOff, pPg->aData, pFS->nMetaRwSize - ); - } -#ifndef NDEBUG - /* pPg->aData causes an uninitialized access via a downstream write(). - After discussion on this list, this memory should not, for performance - reasons, be memset. However, tracking down "real" misuse is more - difficult with this "false" positive, so it is set when NDEBUG. - */ - else if( rc==LSM_OK ){ - memset( pPg->aData, 0x77, pFS->nMetasize ); - } -#endif - } - - if( rc!=LSM_OK ){ - if( pFS->nMapLimit==0 ) lsmFree(pFS->pEnv, pPg->aData); - lsmFree(pFS->pEnv, pPg); - pPg = 0; - }else{ - pPg->iPg = iPg; - pPg->bWrite = bWrite; - pPg->pFS = pFS; - } - } - - *ppPg = pPg; - return rc; -} - -/* -** Release a meta-page reference obtained via a call to lsmFsMetaPageGet(). -*/ -int lsmFsMetaPageRelease(MetaPage *pPg){ - int rc = LSM_OK; - if( pPg ){ - FileSystem *pFS = pPg->pFS; - - if( pFS->nMapLimit==0 ){ - if( pPg->bWrite ){ - i64 iOff = (pPg->iPg==2 ? pFS->nMetasize : 0); - int nWrite = pFS->nMetaRwSize; - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, pPg->aData, nWrite); - } - lsmFree(pFS->pEnv, pPg->aData); - } - - lsmFree(pFS->pEnv, pPg); - } - return rc; -} - -/* -** Return a pointer to a buffer containing the data associated with the -** meta-page passed as the first argument. If parameter pnData is not NULL, -** set *pnData to the size of the meta-page in bytes before returning. -*/ -u8 *lsmFsMetaPageData(MetaPage *pPg, int *pnData){ - if( pnData ) *pnData = pPg->pFS->nMetaRwSize; - return pPg->aData; -} - -/* -** Return true if page is currently writable. This is used in assert() -** statements only. -*/ -#ifndef NDEBUG -int lsmFsPageWritable(Page *pPg){ - return (pPg->flags & PAGE_DIRTY) ? 1 : 0; -} -#endif - -/* -** This is called when block iFrom is being redirected to iTo. If page -** number (*piPg) lies on block iFrom, then calculate the equivalent -** page on block iTo and set *piPg to this value before returning. -*/ -static void fsMovePage( - FileSystem *pFS, /* File system object */ - int iTo, /* Destination block */ - int iFrom, /* Source block */ - LsmPgno *piPg /* IN/OUT: Page number */ -){ - LsmPgno iPg = *piPg; - if( iFrom==fsPageToBlock(pFS, iPg) ){ - const int nPagePerBlock = ( - pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) - ); - *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; - } -} - -/* -** Copy the contents of block iFrom to block iTo. -** -** It is safe to assume that there are no outstanding references to pages -** on block iTo. And that block iFrom is not currently being written. In -** other words, the data can be read and written directly. -*/ -int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){ - Snapshot *p = pFS->pDb->pWorker; - int rc = LSM_OK; - int i; - i64 nMap; - - i64 iFromOff = (i64)(iFrom-1) * pFS->nBlocksize; - i64 iToOff = (i64)(iTo-1) * pFS->nBlocksize; - - assert( iTo!=1 ); - assert( iFrom>iTo ); - - /* Grow the mapping as required. */ - nMap = LSM_MIN(pFS->nMapLimit, (i64)iFrom * pFS->nBlocksize); - fsGrowMapping(pFS, nMap, &rc); - - if( rc==LSM_OK ){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - int nSz = pFS->nPagesize; - u8 *aBuf = 0; - u8 *aData = 0; - - for(i=0; rc==LSM_OK && i<nPagePerBlock; i++){ - i64 iOff = iFromOff + i*nSz; - - /* Set aData to point to a buffer containing the from page */ - if( (iOff+nSz)<=pFS->nMapLimit ){ - u8 *aMap = (u8 *)(pFS->pMap); - aData = &aMap[iOff]; - }else{ - if( aBuf==0 ){ - aBuf = (u8 *)lsmMallocRc(pFS->pEnv, nSz, &rc); - if( aBuf==0 ) break; - } - aData = aBuf; - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); - } - - /* Copy aData to the to page */ - if( rc==LSM_OK ){ - iOff = iToOff + i*nSz; - if( (iOff+nSz)<=pFS->nMapLimit ){ - u8 *aMap = (u8 *)(pFS->pMap); - memcpy(&aMap[iOff], aData, nSz); - }else{ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); - } - } - } - lsmFree(pFS->pEnv, aBuf); - lsmFsPurgeCache(pFS); - } - - /* Update append-point list if necessary */ - for(i=0; i<LSM_APPLIST_SZ; i++){ - fsMovePage(pFS, iTo, iFrom, &p->aiAppend[i]); - } - - /* Update the Segment structure itself */ - fsMovePage(pFS, iTo, iFrom, &pSeg->iFirst); - fsMovePage(pFS, iTo, iFrom, &pSeg->iLastPg); - fsMovePage(pFS, iTo, iFrom, &pSeg->iRoot); - - return rc; -} - -/* -** Append raw data to a segment. Return the database file offset that the -** data is written to (this may be used as the page number if the data -** being appended is a new page record). -** -** This function is only used in compressed database mode. -*/ -static LsmPgno fsAppendData( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Segment to append to */ - const u8 *aData, /* Buffer containing data to write */ - int nData, /* Size of buffer aData[] in bytes */ - int *pRc /* IN/OUT: Error code */ -){ - LsmPgno iRet = 0; - int rc = *pRc; - assert( pFS->pCompress ); - if( rc==LSM_OK ){ - int nRem = 0; - int nWrite = 0; - LsmPgno iLastOnBlock; - LsmPgno iApp = pSeg->iLastPg+1; - - /* If this is the first data written into the segment, find an append-point - ** or allocate a new block. */ - if( iApp==1 ){ - pSeg->iFirst = iApp = findAppendPoint(pFS, 0); - if( iApp==0 ){ - int iBlk; - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - pSeg->iFirst = iApp = fsFirstPageOnBlock(pFS, iBlk); - } - } - iRet = iApp; - - /* Write as much data as is possible at iApp (usually all of it). */ - iLastOnBlock = fsLastPageOnPagesBlock(pFS, iApp); - if( rc==LSM_OK ){ - int nSpace = (int)(iLastOnBlock - iApp + 1); - nWrite = LSM_MIN(nData, nSpace); - nRem = nData - nWrite; - assert( nWrite>=0 ); - if( nWrite!=0 ){ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aData, nWrite); - } - iApp += nWrite; - } - - /* If required, allocate a new block and write the rest of the data - ** into it. Set the next and previous block pointers to link the new - ** block to the old. */ - assert( nRem<=0 || (iApp-1)==iLastOnBlock ); - if( rc==LSM_OK && (iApp-1)==iLastOnBlock ){ - u8 aPtr[4]; /* Space to serialize a u32 */ - int iBlk; /* New block number */ - - if( nWrite>0 ){ - /* Allocate a new block. */ - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - - /* Set the "next" pointer on the old block */ - if( rc==LSM_OK ){ - assert( iApp==(fsPageToBlock(pFS, iApp)*pFS->nBlocksize)-4 ); - lsmPutU32(aPtr, iBlk); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aPtr, sizeof(aPtr)); - } - - /* Set the "prev" pointer on the new block */ - if( rc==LSM_OK ){ - LsmPgno iWrite; - lsmPutU32(aPtr, fsPageToBlock(pFS, iApp)); - iWrite = fsFirstPageOnBlock(pFS, iBlk); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr)); - if( nRem>0 ) iApp = iWrite; - } - }else{ - /* The next block is already allocated. */ - assert( nRem>0 ); - assert( pSeg->pRedirect==0 ); - rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iApp), &iBlk); - iRet = iApp = fsFirstPageOnBlock(pFS, iBlk); - } - - /* Write the remaining data into the new block */ - if( rc==LSM_OK && nRem>0 ){ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, &aData[nWrite], nRem); - iApp += nRem; - } - } - - pSeg->iLastPg = iApp-1; - *pRc = rc; - } - - return iRet; -} - -/* -** This function is only called in compressed database mode. It -** compresses the contents of page pPg and writes the result to the -** buffer at pFS->aOBuffer. The size of the compressed data is stored in -** pPg->nCompress. -** -** If buffer pFS->aOBuffer[] has not been allocated then this function -** allocates it. If this fails, LSM_NOMEM is returned. Otherwise, LSM_OK. -*/ -static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){ - lsm_compress *p = pFS->pCompress; - - if( fsAllocateBuffer(pFS, 1) ) return LSM_NOMEM; - assert( pPg->nData==pFS->nPagesize ); - - pPg->nCompress = pFS->nBuffer; - return p->xCompress(p->pCtx, - (char *)pFS->aOBuffer, &pPg->nCompress, - (const char *)pPg->aData, pPg->nData - ); -} - -/* -** Append a new page to segment pSeg. Set output variable *piNew to the -** page number of the new page before returning. -** -** If the new page is the last on its block, then the 'next' block that -** will be used by the segment is allocated here too. In this case output -** variable *piNext is set to the block number of the next block. -** -** If the new page is the first on its block but not the first in the -** entire segment, set output variable *piPrev to the block number of -** the previous block in the segment. -** -** LSM_OK is returned if successful, or an lsm error code otherwise. If -** any value other than LSM_OK is returned, then the final value of all -** output variables is undefined. -*/ -static int fsAppendPage( - FileSystem *pFS, - Segment *pSeg, - LsmPgno *piNew, - int *piPrev, - int *piNext -){ - LsmPgno iPrev = pSeg->iLastPg; - int rc; - assert( iPrev!=0 ); - - *piPrev = 0; - *piNext = 0; - - if( fsIsLast(pFS, iPrev) ){ - /* Grab the first page on the next block (which has already be - ** allocated). In this case set *piPrev to tell the caller to set - ** the "previous block" pointer in the first 4 bytes of the page. - */ - int iNext; - int iBlk = fsPageToBlock(pFS, iPrev); - assert( pSeg->pRedirect==0 ); - rc = fsBlockNext(pFS, 0, iBlk, &iNext); - if( rc!=LSM_OK ) return rc; - *piNew = fsFirstPageOnBlock(pFS, iNext); - *piPrev = iBlk; - }else{ - *piNew = iPrev+1; - if( fsIsLast(pFS, *piNew) ){ - /* Allocate the next block here. */ - int iBlk; - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - if( rc!=LSM_OK ) return rc; - *piNext = iBlk; - } - } - - pSeg->nSize++; - pSeg->iLastPg = *piNew; - return LSM_OK; -} - -/* -** Flush all pages in the FileSystem.pWaiting list to disk. -*/ -void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){ - int rc = *pRc; - Page *pPg; - - pPg = pFS->pWaiting; - pFS->pWaiting = 0; - - while( pPg ){ - Page *pNext = pPg->pWaitingNext; - if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg); - assert( pPg->nRef==1 ); - lsmFsPageRelease(pPg); - pPg = pNext; - } - *pRc = rc; -} - -/* -** If there exists a hash-table entry associated with page iPg, remove it. -*/ -static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){ - Page *p; - int iHash = fsHashKey(pFS->nHash, iPg); - - for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext); - - if( p ){ - assert( p->nRef==0 || (p->flags & PAGE_FREE)==0 ); - fsPageRemoveFromHash(pFS, p); - p->iPg = 0; - iHash = fsHashKey(pFS->nHash, 0); - p->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = p; - } -} - -/* -** If the page passed as an argument is dirty, update the database file -** (or mapping of the database file) with its current contents and mark -** the page as clean. -** -** Return LSM_OK if the operation is a success, or an LSM error code -** otherwise. -*/ -int lsmFsPagePersist(Page *pPg){ - int rc = LSM_OK; - if( pPg && (pPg->flags & PAGE_DIRTY) ){ - FileSystem *pFS = pPg->pFS; - - if( pFS->pCompress ){ - int iHash; /* Hash key of assigned page number */ - u8 aSz[3]; /* pPg->nCompress as a 24-bit big-endian */ - assert( pPg->pSeg && pPg->iPg==0 && pPg->nCompress==0 ); - - /* Compress the page image. */ - rc = fsCompressIntoBuffer(pFS, pPg); - - /* Serialize the compressed size into buffer aSz[] */ - putRecordSize(aSz, pPg->nCompress, 0); - - /* Write the serialized page record into the database file. */ - pPg->iPg = fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); - fsAppendData(pFS, pPg->pSeg, pFS->aOBuffer, pPg->nCompress, &rc); - fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); - - /* Now that it has a page number, insert the page into the hash table */ - iHash = fsHashKey(pFS->nHash, pPg->iPg); - pPg->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = pPg; - - pPg->pSeg->nSize += (sizeof(aSz) * 2) + pPg->nCompress; - - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; - }else{ - - if( pPg->iPg==0 ){ - /* No page number has been assigned yet. This occurs with pages used - ** in the b-tree hierarchy. They were not assigned page numbers when - ** they were created as doing so would cause this call to - ** lsmFsPagePersist() to write an out-of-order page. Instead a page - ** number is assigned here so that the page data will be appended - ** to the current segment. - */ - Page **pp; - int iPrev = 0; - int iNext = 0; - int iHash; - - assert( pPg->pSeg->iFirst ); - assert( pPg->flags & PAGE_FREE ); - assert( (pPg->flags & PAGE_HASPREV)==0 ); - assert( pPg->nData==pFS->nPagesize-4 ); - - rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext); - if( rc!=LSM_OK ) return rc; - - assert( pPg->flags & PAGE_FREE ); - iHash = fsHashKey(pFS->nHash, pPg->iPg); - fsRemoveHashEntry(pFS, pPg->iPg); - pPg->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = pPg; - assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg ); - - if( iPrev ){ - assert( iNext==0 ); - memmove(&pPg->aData[4], pPg->aData, pPg->nData); - lsmPutU32(pPg->aData, iPrev); - pPg->flags |= PAGE_HASPREV; - pPg->aData += 4; - }else if( iNext ){ - assert( iPrev==0 ); - lsmPutU32(&pPg->aData[pPg->nData], iNext); - }else{ - int nData = pPg->nData; - pPg->nData += 4; - lsmSortedExpandBtreePage(pPg, nData); - } - - pPg->nRef++; - for(pp=&pFS->pWaiting; *pp; pp=&(*pp)->pWaitingNext); - *pp = pPg; - assert( pPg->pWaitingNext==0 ); - - }else{ - i64 iOff; /* Offset to write within database file */ - - iOff = (i64)pFS->nPagesize * (i64)(pPg->iPg-1); - if( fsMmapPage(pFS, pPg->iPg)==0 ){ - u8 *aData = pPg->aData - (pPg->flags & PAGE_HASPREV); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, pFS->nPagesize); - }else if( pPg->flags & PAGE_FREE ){ - fsGrowMapping(pFS, iOff + pFS->nPagesize, &rc); - if( rc==LSM_OK ){ - u8 *aTo = &((u8 *)(pFS->pMap))[iOff]; - u8 *aFrom = pPg->aData - (pPg->flags & PAGE_HASPREV); - memcpy(aTo, aFrom, pFS->nPagesize); - lsmFree(pFS->pEnv, aFrom); - pFS->nCacheAlloc--; - pPg->aData = aTo + (pPg->flags & PAGE_HASPREV); - pPg->flags &= ~PAGE_FREE; - fsPageRemoveFromHash(pFS, pPg); - pPg->pMappedNext = pFS->pMapped; - pFS->pMapped = pPg; - } - } - - lsmFsFlushWaiting(pFS, &rc); - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; - } - } - } - - return rc; -} - -/* -** For non-compressed databases, this function is a no-op. For compressed -** databases, it adds a padding record to the segment passed as the third -** argument. -** -** The size of the padding records is selected so that the last byte -** written is the last byte of a disk sector. This means that if a -** snapshot is taken and checkpointed, subsequent worker processes will -** not write to any sector that contains checkpointed data. -*/ -int lsmFsSortedPadding( - FileSystem *pFS, - Snapshot *pSnapshot, - Segment *pSeg -){ - int rc = LSM_OK; - if( pFS->pCompress && pSeg->iFirst ){ - LsmPgno iLast2; - LsmPgno iLast = pSeg->iLastPg; /* Current last page of segment */ - int nPad; /* Bytes of padding required */ - u8 aSz[3]; - - iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1; - assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) ); - nPad = (int)(iLast2 - iLast); - - if( iLast2>fsLastPageOnPagesBlock(pFS, iLast) ){ - nPad -= 4; - } - assert( nPad>=0 ); - - if( nPad>=6 ){ - pSeg->nSize += nPad; - nPad -= 6; - putRecordSize(aSz, nPad, 1); - fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); - memset(pFS->aOBuffer, 0, nPad); - fsAppendData(pFS, pSeg, pFS->aOBuffer, nPad, &rc); - fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); - }else if( nPad>0 ){ - u8 aBuf[5] = {0,0,0,0,0}; - aBuf[0] = (u8)nPad; - aBuf[nPad-1] = (u8)nPad; - fsAppendData(pFS, pSeg, aBuf, nPad, &rc); - } - - assert( rc!=LSM_OK - || pSeg->iLastPg==fsLastPageOnPagesBlock(pFS, pSeg->iLastPg) - || ((pSeg->iLastPg + 1) % pFS->szSector)==0 - ); - } - - return rc; -} - - -/* -** Increment the reference count on the page object passed as the first -** argument. -*/ -void lsmFsPageRef(Page *pPg){ - if( pPg ){ - pPg->nRef++; - } -} - -/* -** Release a page-reference obtained using fsPageGet(). -*/ -int lsmFsPageRelease(Page *pPg){ - int rc = LSM_OK; - if( pPg ){ - assert( pPg->nRef>0 ); - pPg->nRef--; - if( pPg->nRef==0 ){ - FileSystem *pFS = pPg->pFS; - rc = lsmFsPagePersist(pPg); - pFS->nOut--; - - assert( pPg->pFS->pCompress - || fsIsFirst(pPg->pFS, pPg->iPg)==0 - || (pPg->flags & PAGE_HASPREV) - ); - pPg->aData -= (pPg->flags & PAGE_HASPREV); - pPg->flags &= ~PAGE_HASPREV; - - if( (pPg->flags & PAGE_FREE)==0 ){ - /* Removed from mapped list */ - Page **pp; - for(pp=&pFS->pMapped; (*pp)!=pPg; pp=&(*pp)->pMappedNext); - *pp = pPg->pMappedNext; - pPg->pMappedNext = 0; - - /* Add to free list */ - pPg->pFreeNext = pFS->pFree; - pFS->pFree = pPg; - }else{ - fsPageAddToLru(pFS, pPg); - } - } - } - - return rc; -} - -/* -** Return the total number of pages read from the database file. -*/ -int lsmFsNRead(FileSystem *pFS){ return pFS->nRead; } - -/* -** Return the total number of pages written to the database file. -*/ -int lsmFsNWrite(FileSystem *pFS){ return pFS->nWrite; } - -/* -** Return a copy of the environment pointer used by the file-system object. -*/ -lsm_env *lsmFsEnv(FileSystem *pFS){ - return pFS->pEnv; -} - -/* -** Return a copy of the environment pointer used by the file-system object -** to which this page belongs. -*/ -lsm_env *lsmPageEnv(Page *pPg) { - return pPg->pFS->pEnv; -} - -/* -** Return a pointer to the file-system object associated with the Page -** passed as the only argument. -*/ -FileSystem *lsmPageFS(Page *pPg){ - return pPg->pFS; -} - -/* -** Return the sector-size as reported by the log file handle. -*/ -int lsmFsSectorSize(FileSystem *pFS){ - return pFS->szSector; -} - -/* -** Helper function for lsmInfoArrayStructure(). -*/ -static Segment *startsWith(Segment *pRun, LsmPgno iFirst){ - return (iFirst==pRun->iFirst) ? pRun : 0; -} - -/* -** Return the segment that starts with page iFirst, if any. If no such segment -** can be found, return NULL. -*/ -static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){ - Level *pLvl; /* Used to iterate through db levels */ - Segment *pSeg = 0; /* Pointer to segment to return */ - - for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){ - if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){ - int i; - for(i=0; i<pLvl->nRight; i++){ - if( (pSeg = startsWith(&pLvl->aRhs[i], iFirst)) ) break; - } - } - } - - return pSeg; -} - -/* -** This function implements the lsm_info(LSM_INFO_ARRAY_STRUCTURE) request. -** If successful, *pzOut is set to point to a nul-terminated string -** containing the array structure and LSM_OK is returned. The caller should -** eventually free the string using lsmFree(). -** -** If an error occurs, *pzOut is set to NULL and an LSM error code returned. -*/ -int lsmInfoArrayStructure( - lsm_db *pDb, - int bBlock, /* True for block numbers only */ - LsmPgno iFirst, - char **pzOut -){ - int rc = LSM_OK; - Snapshot *pWorker; /* Worker snapshot */ - Segment *pArray = 0; /* Array to report on */ - int bUnlock = 0; - - *pzOut = 0; - if( iFirst==0 ) return LSM_ERROR; - - /* Obtain the worker snapshot */ - pWorker = pDb->pWorker; - if( !pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - bUnlock = 1; - } - - /* Search for the array that starts on page iFirst */ - pArray = findSegment(pWorker, iFirst); - - if( pArray==0 ){ - /* Could not find the requested array. This is an error. */ - rc = LSM_ERROR; - }else{ - FileSystem *pFS = pDb->pFS; - LsmString str; - int iBlk; - int iLastBlk; - - iBlk = fsPageToBlock(pFS, pArray->iFirst); - iLastBlk = fsPageToBlock(pFS, pArray->iLastPg); - - lsmStringInit(&str, pDb->pEnv); - if( bBlock ){ - lsmStringAppendf(&str, "%d", iBlk); - while( iBlk!=iLastBlk ){ - fsBlockNext(pFS, pArray, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", iBlk); - } - }else{ - lsmStringAppendf(&str, "%d", pArray->iFirst); - while( iBlk!=iLastBlk ){ - lsmStringAppendf(&str, " %d", fsLastPageOnBlock(pFS, iBlk)); - fsBlockNext(pFS, pArray, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", fsFirstPageOnBlock(pFS, iBlk)); - } - lsmStringAppendf(&str, " %d", pArray->iLastPg); - } - - *pzOut = str.z; - } - - if( bUnlock ){ - int rcwork = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcwork); - } - return rc; -} - -int lsmFsSegmentContainsPg( - FileSystem *pFS, - Segment *pSeg, - LsmPgno iPg, - int *pbRes -){ - Redirect *pRedir = pSeg->pRedirect; - int rc = LSM_OK; - int iBlk; - int iLastBlk; - int iPgBlock; /* Block containing page iPg */ - - iPgBlock = fsPageToBlock(pFS, pSeg->iFirst); - iBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iFirst)); - iLastBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iLastPg)); - - while( iBlk!=iLastBlk && iBlk!=iPgBlock && rc==LSM_OK ){ - rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); - } - - *pbRes = (iBlk==iPgBlock); - return rc; -} - -/* -** This function implements the lsm_info(LSM_INFO_ARRAY_PAGES) request. -** If successful, *pzOut is set to point to a nul-terminated string -** containing the array structure and LSM_OK is returned. The caller should -** eventually free the string using lsmFree(). -** -** If an error occurs, *pzOut is set to NULL and an LSM error code returned. -*/ -int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){ - int rc = LSM_OK; - Snapshot *pWorker; /* Worker snapshot */ - Segment *pSeg = 0; /* Array to report on */ - int bUnlock = 0; - - *pzOut = 0; - if( iFirst==0 ) return LSM_ERROR; - - /* Obtain the worker snapshot */ - pWorker = pDb->pWorker; - if( !pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - bUnlock = 1; - } - - /* Search for the array that starts on page iFirst */ - pSeg = findSegment(pWorker, iFirst); - - if( pSeg==0 ){ - /* Could not find the requested array. This is an error. */ - rc = LSM_ERROR; - }else{ - Page *pPg = 0; - FileSystem *pFS = pDb->pFS; - LsmString str; - - lsmStringInit(&str, pDb->pEnv); - rc = lsmFsDbPageGet(pFS, pSeg, iFirst, &pPg); - while( rc==LSM_OK && pPg ){ - Page *pNext = 0; - lsmStringAppendf(&str, " %lld", lsmFsPageNumber(pPg)); - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - - if( rc!=LSM_OK ){ - lsmFree(pDb->pEnv, str.z); - }else{ - *pzOut = str.z; - } - } - - if( bUnlock ){ - int rcwork = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcwork); - } - return rc; -} - -/* -** The following macros are used by the integrity-check code. Associated with -** each block in the database is an 8-bit bit mask (the entry in the aUsed[] -** array). As the integrity-check meanders through the database, it sets the -** following bits to indicate how each block is used. -** -** INTEGRITY_CHECK_FIRST_PG: -** First page of block is in use by sorted run. -** -** INTEGRITY_CHECK_LAST_PG: -** Last page of block is in use by sorted run. -** -** INTEGRITY_CHECK_USED: -** At least one page of the block is in use by a sorted run. -** -** INTEGRITY_CHECK_FREE: -** The free block list contains an entry corresponding to this block. -*/ -#define INTEGRITY_CHECK_FIRST_PG 0x01 -#define INTEGRITY_CHECK_LAST_PG 0x02 -#define INTEGRITY_CHECK_USED 0x04 -#define INTEGRITY_CHECK_FREE 0x08 - -/* -** Helper function for lsmFsIntegrityCheck() -*/ -static void checkBlocks( - FileSystem *pFS, - Segment *pSeg, - int bExtra, /* If true, count the "next" block if any */ - int nUsed, - u8 *aUsed -){ - if( pSeg ){ - if( pSeg && pSeg->nSize>0 ){ - int rc; - int iBlk; /* Current block (during iteration) */ - int iLastBlk; /* Last block of segment */ - int iFirstBlk; /* First block of segment */ - int bLastIsLastOnBlock; /* True iLast is the last on its block */ - - assert( 0==fsSegmentRedirects(pFS, pSeg) ); - iBlk = iFirstBlk = fsPageToBlock(pFS, pSeg->iFirst); - iLastBlk = fsPageToBlock(pFS, pSeg->iLastPg); - - bLastIsLastOnBlock = (fsLastPageOnBlock(pFS, iLastBlk)==pSeg->iLastPg); - assert( iBlk>0 ); - - do { - /* iBlk is a part of this sorted run. */ - aUsed[iBlk-1] |= INTEGRITY_CHECK_USED; - - /* If the first page of this block is also part of the segment, - ** set the flag to indicate that the first page of iBlk is in use. - */ - if( fsFirstPageOnBlock(pFS, iBlk)==pSeg->iFirst || iBlk!=iFirstBlk ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_FIRST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_FIRST_PG; - } - - /* Unless the sorted run finishes before the last page on this block, - ** the last page of this block is also in use. */ - if( iBlk!=iLastBlk || bLastIsLastOnBlock ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_LAST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_LAST_PG; - } - - /* Special case. The sorted run being scanned is the output run of - ** a level currently undergoing an incremental merge. The sorted - ** run ends on the last page of iBlk, but the next block has already - ** been allocated. So mark it as in use as well. */ - if( iBlk==iLastBlk && bLastIsLastOnBlock && bExtra ){ - int iExtra = 0; - rc = fsBlockNext(pFS, pSeg, iBlk, &iExtra); - assert( rc==LSM_OK ); - - assert( aUsed[iExtra-1]==0 ); - aUsed[iExtra-1] |= INTEGRITY_CHECK_USED; - aUsed[iExtra-1] |= INTEGRITY_CHECK_FIRST_PG; - aUsed[iExtra-1] |= INTEGRITY_CHECK_LAST_PG; - } - - /* Move on to the next block in the sorted run. Or set iBlk to zero - ** in order to break out of the loop if this was the last block in - ** the run. */ - if( iBlk==iLastBlk ){ - iBlk = 0; - }else{ - rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); - assert( rc==LSM_OK ); - } - }while( iBlk ); - } - } -} - -typedef struct CheckFreelistCtx CheckFreelistCtx; -struct CheckFreelistCtx { - u8 *aUsed; - int nBlock; -}; -static int checkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - CheckFreelistCtx *p = (CheckFreelistCtx *)pCtx; - - assert( iBlk>=1 ); - assert( iBlk<=p->nBlock ); - assert( p->aUsed[iBlk-1]==0 ); - p->aUsed[iBlk-1] = INTEGRITY_CHECK_FREE; - return 0; -} - -/* -** This function checks that all blocks in the database file are accounted -** for. For each block, exactly one of the following must be true: -** -** + the block is part of a sorted run, or -** + the block is on the free-block list -** -** This function also checks that there are no references to blocks with -** out-of-range block numbers. -** -** If no errors are found, non-zero is returned. If an error is found, an -** assert() fails. -*/ -int lsmFsIntegrityCheck(lsm_db *pDb){ - CheckFreelistCtx ctx; - FileSystem *pFS = pDb->pFS; - int i; - int rc; - Freelist freelist = {0, 0, 0}; - u8 *aUsed; - Level *pLevel; - Snapshot *pWorker = pDb->pWorker; - int nBlock = pWorker->nBlock; - -#if 0 - static int nCall = 0; - nCall++; - printf("%d calls\n", nCall); -#endif - - aUsed = lsmMallocZero(pDb->pEnv, nBlock); - if( aUsed==0 ){ - /* Malloc has failed. Since this function is only called within debug - ** builds, this probably means the user is running an OOM injection test. - ** Regardless, it will not be possible to run the integrity-check at this - ** time, so assume the database is Ok and return non-zero. */ - return 1; - } - - for(pLevel=pWorker->pLevel; pLevel; pLevel=pLevel->pNext){ - int j; - checkBlocks(pFS, &pLevel->lhs, (pLevel->nRight!=0), nBlock, aUsed); - for(j=0; j<pLevel->nRight; j++){ - checkBlocks(pFS, &pLevel->aRhs[j], 0, nBlock, aUsed); - } - } - - /* Mark all blocks in the free-list as used */ - ctx.aUsed = aUsed; - ctx.nBlock = nBlock; - rc = lsmWalkFreelist(pDb, 0, checkFreelistCb, (void *)&ctx); - - if( rc==LSM_OK ){ - for(i=0; i<nBlock; i++) assert( aUsed[i]!=0 ); - } - - lsmFree(pDb->pEnv, aUsed); - lsmFree(pDb->pEnv, freelist.aEntry); - - return 1; -} - -#ifndef NDEBUG -/* -** Return true if pPg happens to be the last page in segment pSeg. Or false -** otherwise. This function is only invoked as part of assert() conditions. -*/ -int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){ - if( pPg->pFS->pCompress ){ - LsmPgno iNext = 0; - int rc; - rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext); - return (rc!=LSM_OK || iNext==0); - } - return (pPg->iPg==pSeg->iLastPg); -} -#endif diff --git a/ext/lsm1/lsm_log.c b/ext/lsm1/lsm_log.c deleted file mode 100644 index 3dcef42f7..000000000 --- a/ext/lsm1/lsm_log.c +++ /dev/null @@ -1,1156 +0,0 @@ -/* -** 2011-08-13 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the implementation of LSM database logging. Logging -** has one purpose in LSM - to make transactions durable. -** -** When data is written to an LSM database, it is initially stored in an -** in-memory tree structure. Since this structure is in volatile memory, -** if a power failure or application crash occurs it may be lost. To -** prevent loss of data in this case, each time a record is written to the -** in-memory tree an equivalent record is appended to the log on disk. -** If a power failure or application crash does occur, data can be recovered -** by reading the log. -** -** A log file consists of the following types of records representing data -** written into the database: -** -** LOG_WRITE: A key-value pair written to the database. -** LOG_DELETE: A delete key issued to the database. -** LOG_COMMIT: A transaction commit. -** -** And the following types of records for ancillary purposes.. -** -** LOG_EOF: A record indicating the end of a log file. -** LOG_PAD1: A single byte padding record. -** LOG_PAD2: An N byte padding record (N>1). -** LOG_JUMP: A pointer to another offset within the log file. -** -** Each transaction written to the log contains one or more LOG_WRITE and/or -** LOG_DELETE records, followed by a LOG_COMMIT record. The LOG_COMMIT record -** contains an 8-byte checksum based on all previous data written to the -** log file. -** -** LOG CHECKSUMS & RECOVERY -** -** Checksums are found in two types of log records: LOG_COMMIT and -** LOG_CKSUM records. In order to recover content from a log, a client -** reads each record from the start of the log, calculating a checksum as -** it does. Each time a LOG_COMMIT or LOG_CKSUM is encountered, the -** recovery process verifies that the checksum stored in the log -** matches the calculated checksum. If it does not, the recovery process -** can stop reading the log. -** -** If a recovery process reads records (other than COMMIT or CKSUM) -** consisting of at least LSM_CKSUM_MAXDATA bytes, then the next record in -** the log must be either a LOG_CKSUM or LOG_COMMIT record. If it is -** not, the recovery process also stops reading the log. -** -** To recover the log file, it must be read twice. The first time to -** determine the location of the last valid commit record. And the second -** time to load data into the in-memory tree. -** -** Todo: Surely there is a better way... -** -** LOG WRAPPING -** -** If the log file were never deleted or wrapped, it would be possible to -** read it from start to end each time is required recovery (i.e each time -** the number of database clients changes from 0 to 1). Effectively reading -** the entire history of the database each time. This would quickly become -** inefficient. Additionally, since the log file would grow without bound, -** it wastes storage space. -** -** Instead, part of each checkpoint written into the database file contains -** a log offset (and other information required to read the log starting at -** at this offset) at which to begin recovery. Offset $O. -** -** Once a checkpoint has been written and synced into the database file, it -** is guaranteed that no recovery process will need to read any data before -** offset $O of the log file. It is therefore safe to begin overwriting -** any data that occurs before offset $O. -** -** This implementation separates the log into three regions mapped into -** the log file - regions 0, 1 and 2. During recovery, regions are read -** in ascending order (i.e. 0, then 1, then 2). Each region is zero or -** more bytes in size. -** -** |---1---|..|--0--|.|--2--|.... -** -** New records are always appended to the end of region 2. -** -** Initially (when it is empty), all three regions are zero bytes in size. -** Each of them are located at the beginning of the file. As records are -** added to the log, region 2 grows, so that the log consists of a zero -** byte region 1, followed by a zero byte region 0, followed by an N byte -** region 2. After one or more checkpoints have been written to disk, -** the start point of region 2 is moved to $O. For example: -** -** A) ||.........|--2--|.... -** -** (both regions 0 and 1 are 0 bytes in size at offset 0). -** -** Eventually, the log wraps around to write new records into the start. -** At this point, region 2 is renamed to region 0. Region 0 is renamed -** to region 2. After appending a few records to the new region 2, the -** log file looks like this: -** -** B) ||--2--|...|--0--|.... -** -** (region 1 is still 0 bytes in size, located at offset 0). -** -** Any checkpoints made at this point may reduce the size of region 0. -** However, if they do not, and region 2 expands so that it is about to -** overwrite the start of region 0, then region 2 is renamed to region 1, -** and a new region 2 created at the end of the file following the existing -** region 0. -** -** C) |---1---|..|--0--|.|-2-| -** -** In this state records are appended to region 2 until checkpoints have -** contracted regions 0 AND 1 UNTil they are both zero bytes in size. They -** are then shifted to the start of the log file, leaving the system in -** the equivalent of state A above. -** -** Alternatively, state B may transition directly to state A if the size -** of region 0 is reduced to zero bytes before region 2 threatens to -** encroach upon it. -** -** LOG_PAD1 & LOG_PAD2 RECORDS -** -** PAD1 and PAD2 records may appear in a log file at any point. They allow -** a process writing the log file align the beginning of transactions with -** the beginning of disk sectors, which increases robustness. -** -** RECORD FORMATS: -** -** LOG_EOF: * A single 0x00 byte. -** -** LOG_PAD1: * A single 0x01 byte. -** -** LOG_PAD2: * A single 0x02 byte, followed by -** * The number of unused bytes (N) as a varint, -** * An N byte block of unused space. -** -** LOG_COMMIT: * A single 0x03 byte. -** * An 8-byte checksum. -** -** LOG_JUMP: * A single 0x04 byte. -** * Absolute file offset to jump to, encoded as a varint. -** -** LOG_WRITE: * A single 0x06 or 0x07 byte, -** * The number of bytes in the key, encoded as a varint, -** * The number of bytes in the value, encoded as a varint, -** * If the first byte was 0x07, an 8 byte checksum. -** * The key data, -** * The value data. -** -** LOG_DELETE: * A single 0x08 or 0x09 byte, -** * The number of bytes in the key, encoded as a varint, -** * If the first byte was 0x09, an 8 byte checksum. -** * The key data. -** -** Varints are as described in lsm_varint.c (SQLite 4 format). -** -** CHECKSUMS: -** -** The checksum is calculated using two 32-bit unsigned integers, s0 and -** s1. The initial value for both is 42. It is updated each time a record -** is written into the log file by treating the encoded (binary) record as -** an array of 32-bit little-endian integers. Then, if x[] is the integer -** array, updating the checksum accumulators as follows: -** -** for i from 0 to n-1 step 2: -** s0 += x[i] + s1; -** s1 += x[i+1] + s0; -** endfor -** -** If the record is not an even multiple of 8-bytes in size it is padded -** with zeroes to make it so before the checksum is updated. -** -** The checksum stored in a COMMIT, WRITE or DELETE is based on all bytes -** up to the start of the 8-byte checksum itself, including the COMMIT, -** WRITE or DELETE fields that appear before the checksum in the record. -** -** VARINT FORMAT -** -** See lsm_varint.c. -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -/* Log record types */ -#define LSM_LOG_EOF 0x00 -#define LSM_LOG_PAD1 0x01 -#define LSM_LOG_PAD2 0x02 -#define LSM_LOG_COMMIT 0x03 -#define LSM_LOG_JUMP 0x04 - -#define LSM_LOG_WRITE 0x06 -#define LSM_LOG_WRITE_CKSUM 0x07 - -#define LSM_LOG_DELETE 0x08 -#define LSM_LOG_DELETE_CKSUM 0x09 - -#define LSM_LOG_DRANGE 0x0A -#define LSM_LOG_DRANGE_CKSUM 0x0B - -/* Require a checksum every 32KB. */ -#define LSM_CKSUM_MAXDATA (32*1024) - -/* Do not wrap a log file smaller than this in bytes. */ -#define LSM_MIN_LOGWRAP (128*1024) - -/* -** szSector: -** Commit records must be aligned to end on szSector boundaries. If -** the safety-mode is set to NORMAL or OFF, this value is 1. Otherwise, -** if the safety-mode is set to FULL, it is the size of the file-system -** sectors as reported by lsmFsSectorSize(). -*/ -struct LogWriter { - u32 cksum0; /* Checksum 0 at offset iOff */ - u32 cksum1; /* Checksum 1 at offset iOff */ - int iCksumBuf; /* Bytes of buf that have been checksummed */ - i64 iOff; /* Offset at start of buffer buf */ - int szSector; /* Sector size for this transaction */ - LogRegion jump; /* Avoid writing to this region */ - i64 iRegion1End; /* End of first region written by trans */ - i64 iRegion2Start; /* Start of second regions written by trans */ - LsmString buf; /* Buffer containing data not yet written */ -}; - -/* -** Return the result of interpreting the first 4 bytes in buffer aIn as -** a 32-bit unsigned little-endian integer. -*/ -static u32 getU32le(u8 *aIn){ - return ((u32)aIn[3] << 24) - + ((u32)aIn[2] << 16) - + ((u32)aIn[1] << 8) - + ((u32)aIn[0]); -} - - -/* -** This function is the same as logCksum(), except that pointer "a" need -** not be aligned to an 8-byte boundary or padded with zero bytes. This -** version is slower, but sometimes more convenient to use. -*/ -static void logCksumUnaligned( - char *z, /* Input buffer */ - int n, /* Size of input buffer in bytes */ - u32 *pCksum0, /* IN/OUT: Checksum value 1 */ - u32 *pCksum1 /* IN/OUT: Checksum value 2 */ -){ - u8 *a = (u8 *)z; - u32 cksum0 = *pCksum0; - u32 cksum1 = *pCksum1; - int nIn = (n/8) * 8; - int i; - - assert( n>0 ); - for(i=0; i<nIn; i+=8){ - cksum0 += getU32le(&a[i]) + cksum1; - cksum1 += getU32le(&a[i+4]) + cksum0; - } - - if( nIn!=n ){ - u8 aBuf[8] = {0, 0, 0, 0, 0, 0, 0, 0}; - assert( (n-nIn)<8 && n>nIn ); - memcpy(aBuf, &a[nIn], n-nIn); - cksum0 += getU32le(aBuf) + cksum1; - cksum1 += getU32le(&aBuf[4]) + cksum0; - } - - *pCksum0 = cksum0; - *pCksum1 = cksum1; -} - -/* -** Update pLog->cksum0 and pLog->cksum1 so that the first nBuf bytes in the -** write buffer (pLog->buf) are included in the checksum. -*/ -static void logUpdateCksum(LogWriter *pLog, int nBuf){ - assert( (pLog->iCksumBuf % 8)==0 ); - assert( pLog->iCksumBuf<=nBuf ); - assert( (nBuf % 8)==0 || nBuf==pLog->buf.n ); - if( nBuf>pLog->iCksumBuf ){ - logCksumUnaligned( - &pLog->buf.z[pLog->iCksumBuf], nBuf-pLog->iCksumBuf, - &pLog->cksum0, &pLog->cksum1 - ); - } - pLog->iCksumBuf = nBuf; -} - -static i64 firstByteOnSector(LogWriter *pLog, i64 iOff){ - return (iOff / pLog->szSector) * pLog->szSector; -} -static i64 lastByteOnSector(LogWriter *pLog, i64 iOff){ - return firstByteOnSector(pLog, iOff) + pLog->szSector - 1; -} - -/* -** If possible, reclaim log file space. Log file space is reclaimed after -** a snapshot that points to the same data in the database file is synced -** into the db header. -*/ -static int logReclaimSpace(lsm_db *pDb){ - int rc; - int iMeta; - int bRotrans; /* True if there exists some ro-trans */ - - /* Test if there exists some other connection with a read-only transaction - ** open. If there does, then log file space may not be reclaimed. */ - rc = lsmDetectRoTrans(pDb, &bRotrans); - if( rc!=LSM_OK || bRotrans ) return rc; - - iMeta = (int)pDb->pShmhdr->iMetaPage; - if( iMeta==1 || iMeta==2 ){ - DbLog *pLog = &pDb->treehdr.log; - i64 iSyncedId; - - /* Read the snapshot-id of the snapshot stored on meta-page iMeta. Note - ** that in theory, the value read is untrustworthy (due to a race - ** condition - see comments above lsmFsReadSyncedId()). So it is only - ** ever used to conclude that no log space can be reclaimed. If it seems - ** to indicate that it may be possible to reclaim log space, a - ** second call to lsmCheckpointSynced() (which does return trustworthy - ** values) is made below to confirm. */ - rc = lsmFsReadSyncedId(pDb, iMeta, &iSyncedId); - - if( rc==LSM_OK && pLog->iSnapshotId!=iSyncedId ){ - i64 iSnapshotId = 0; - i64 iOff = 0; - rc = lsmCheckpointSynced(pDb, &iSnapshotId, &iOff, 0); - if( rc==LSM_OK && pLog->iSnapshotId<iSnapshotId ){ - int iRegion; - for(iRegion=0; iRegion<3; iRegion++){ - LogRegion *p = &pLog->aRegion[iRegion]; - if( iOff>=p->iStart && iOff<=p->iEnd ) break; - p->iStart = 0; - p->iEnd = 0; - } - assert( iRegion<3 ); - pLog->aRegion[iRegion].iStart = iOff; - pLog->iSnapshotId = iSnapshotId; - } - } - } - return rc; -} - -/* -** This function is called when a write-transaction is first opened. It -** is assumed that the caller is holding the client-mutex when it is -** called. -** -** Before returning, this function allocates the LogWriter object that -** will be used to write to the log file during the write transaction. -** LSM_OK is returned if no error occurs, otherwise an LSM error code. -*/ -int lsmLogBegin(lsm_db *pDb){ - int rc = LSM_OK; - LogWriter *pNew; - LogRegion *aReg; - - if( pDb->bUseLog==0 ) return LSM_OK; - - /* If the log file has not yet been opened, open it now. Also allocate - ** the LogWriter structure, if it has not already been allocated. */ - rc = lsmFsOpenLog(pDb, 0); - if( pDb->pLogWriter==0 ){ - pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); - if( pNew ){ - lsmStringInit(&pNew->buf, pDb->pEnv); - rc = lsmStringExtend(&pNew->buf, 2); - } - pDb->pLogWriter = pNew; - }else{ - pNew = pDb->pLogWriter; - assert( (u8 *)(&pNew[1])==(u8 *)(&((&pNew->buf)[1])) ); - memset(pNew, 0, ((u8 *)&pNew->buf) - (u8 *)pNew); - pNew->buf.n = 0; - } - - if( rc==LSM_OK ){ - /* The following call detects whether or not a new snapshot has been - ** synced into the database file. If so, it updates the contents of - ** the pDb->treehdr.log structure to reclaim any space in the log - ** file that is no longer required. - ** - ** TODO: Calling this every transaction is overkill. And since the - ** call has to read and checksum a snapshot from the database file, - ** it is expensive. It would be better to figure out a way so that - ** this is only called occasionally - say for every 32KB written to - ** the log file. - */ - rc = logReclaimSpace(pDb); - } - if( rc!=LSM_OK ){ - lsmLogClose(pDb); - return rc; - } - - /* Set the effective sector-size for this transaction. Sectors are assumed - ** to be one byte in size if the safety-mode is OFF or NORMAL, or as - ** reported by lsmFsSectorSize if it is FULL. */ - if( pDb->eSafety==LSM_SAFETY_FULL ){ - pNew->szSector = lsmFsSectorSize(pDb->pFS); - assert( pNew->szSector>0 ); - }else{ - pNew->szSector = 1; - } - - /* There are now three scenarios: - ** - ** 1) Regions 0 and 1 are both zero bytes in size and region 2 begins - ** at a file offset greater than LSM_MIN_LOGWRAP. In this case, wrap - ** around to the start and write data into the start of the log file. - ** - ** 2) Region 1 is zero bytes in size and region 2 occurs earlier in the - ** file than region 0. In this case, append data to region 2, but - ** remember to jump over region 1 if required. - ** - ** 3) Region 2 is the last in the file. Append to it. - */ - aReg = &pDb->treehdr.log.aRegion[0]; - - assert( aReg[0].iEnd==0 || aReg[0].iEnd>aReg[0].iStart ); - assert( aReg[1].iEnd==0 || aReg[1].iEnd>aReg[1].iStart ); - - pNew->cksum0 = pDb->treehdr.log.cksum0; - pNew->cksum1 = pDb->treehdr.log.cksum1; - - if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=LSM_MIN_LOGWRAP ){ - /* Case 1. Wrap around to the start of the file. Write an LSM_LOG_JUMP - ** into the log file in this case. Pad it out to 8 bytes using a PAD2 - ** record so that the checksums can be updated immediately. */ - u8 aJump[] = { - LSM_LOG_PAD2, 0x04, 0x00, 0x00, 0x00, 0x00, LSM_LOG_JUMP, 0x00 - }; - - lsmStringBinAppend(&pNew->buf, aJump, sizeof(aJump)); - logUpdateCksum(pNew, pNew->buf.n); - rc = lsmFsWriteLog(pDb->pFS, aReg[2].iEnd, &pNew->buf); - pNew->iCksumBuf = pNew->buf.n = 0; - - aReg[2].iEnd += 8; - pNew->jump = aReg[0] = aReg[2]; - aReg[2].iStart = aReg[2].iEnd = 0; - }else if( aReg[1].iEnd==0 && aReg[2].iEnd<aReg[0].iEnd ){ - /* Case 2. */ - pNew->iOff = aReg[2].iEnd; - pNew->jump = aReg[0]; - }else{ - /* Case 3. */ - assert( aReg[2].iStart>=aReg[0].iEnd && aReg[2].iStart>=aReg[1].iEnd ); - pNew->iOff = aReg[2].iEnd; - } - - if( pNew->jump.iStart ){ - i64 iRound; - assert( pNew->jump.iStart>pNew->iOff ); - - iRound = firstByteOnSector(pNew, pNew->jump.iStart); - if( iRound>pNew->iOff ) pNew->jump.iStart = iRound; - pNew->jump.iEnd = lastByteOnSector(pNew, pNew->jump.iEnd); - } - - assert( pDb->pLogWriter==pNew ); - return rc; -} - -/* -** This function is called when a write-transaction is being closed. -** Parameter bCommit is true if the transaction is being committed, -** or false otherwise. The caller must hold the client-mutex to call -** this function. -** -** A call to this function deletes the LogWriter object allocated by -** lsmLogBegin(). If the transaction is being committed, the shared state -** in *pLog is updated before returning. -*/ -void lsmLogEnd(lsm_db *pDb, int bCommit){ - DbLog *pLog; - LogWriter *p; - p = pDb->pLogWriter; - - if( p==0 ) return; - pLog = &pDb->treehdr.log; - - if( bCommit ){ - pLog->aRegion[2].iEnd = p->iOff; - pLog->cksum0 = p->cksum0; - pLog->cksum1 = p->cksum1; - if( p->iRegion1End ){ - /* This happens when the transaction had to jump over some other - ** part of the log. */ - assert( pLog->aRegion[1].iEnd==0 ); - assert( pLog->aRegion[2].iStart<p->iRegion1End ); - pLog->aRegion[1].iStart = pLog->aRegion[2].iStart; - pLog->aRegion[1].iEnd = p->iRegion1End; - pLog->aRegion[2].iStart = p->iRegion2Start; - } - } -} - -static int jumpIfRequired( - lsm_db *pDb, - LogWriter *pLog, - int nReq, - int *pbJump -){ - /* Determine if it is necessary to add an LSM_LOG_JUMP to jump over the - ** jump region before writing the LSM_LOG_WRITE or DELETE record. This - ** is necessary if there is insufficient room between the current offset - ** and the jump region to fit the new WRITE/DELETE record and the largest - ** possible JUMP record with up to 7 bytes of padding (a total of 17 - ** bytes). */ - if( (pLog->jump.iStart > (pLog->iOff + pLog->buf.n)) - && (pLog->jump.iStart < (pLog->iOff + pLog->buf.n + (nReq + 17))) - ){ - int rc; /* Return code */ - i64 iJump; /* Offset to jump to */ - u8 aJump[10]; /* Encoded jump record */ - int nJump; /* Valid bytes in aJump[] */ - int nPad; /* Bytes of padding required */ - - /* Serialize the JUMP record */ - iJump = pLog->jump.iEnd+1; - aJump[0] = LSM_LOG_JUMP; - nJump = 1 + lsmVarintPut64(&aJump[1], iJump); - - /* Adding padding to the contents of the buffer so that it will be a - ** multiple of 8 bytes in size after the JUMP record is appended. This - ** is not strictly required, it just makes the keeping the running - ** checksum up to date in this file a little simpler. */ - nPad = (pLog->buf.n + nJump) % 8; - if( nPad ){ - u8 aPad[7] = {0,0,0,0,0,0,0}; - nPad = 8-nPad; - if( nPad==1 ){ - aPad[0] = LSM_LOG_PAD1; - }else{ - aPad[0] = LSM_LOG_PAD2; - aPad[1] = (u8)(nPad-2); - } - rc = lsmStringBinAppend(&pLog->buf, aPad, nPad); - if( rc!=LSM_OK ) return rc; - } - - /* Append the JUMP record to the buffer. Then flush the buffer to disk - ** and update the checksums. The next write to the log file (assuming - ** there is no transaction rollback) will be to offset iJump (just past - ** the jump region). */ - rc = lsmStringBinAppend(&pLog->buf, aJump, nJump); - if( rc!=LSM_OK ) return rc; - assert( (pLog->buf.n % 8)==0 ); - rc = lsmFsWriteLog(pDb->pFS, pLog->iOff, &pLog->buf); - if( rc!=LSM_OK ) return rc; - logUpdateCksum(pLog, pLog->buf.n); - pLog->iRegion1End = (pLog->iOff + pLog->buf.n); - pLog->iRegion2Start = iJump; - pLog->iOff = iJump; - pLog->iCksumBuf = pLog->buf.n = 0; - if( pbJump ) *pbJump = 1; - } - - return LSM_OK; -} - -static int logCksumAndFlush(lsm_db *pDb){ - int rc; /* Return code */ - LogWriter *pLog = pDb->pLogWriter; - - /* Calculate the checksum value. Append it to the buffer. */ - logUpdateCksum(pLog, pLog->buf.n); - lsmPutU32((u8 *)&pLog->buf.z[pLog->buf.n], pLog->cksum0); - pLog->buf.n += 4; - lsmPutU32((u8 *)&pLog->buf.z[pLog->buf.n], pLog->cksum1); - pLog->buf.n += 4; - - /* Write the contents of the buffer to disk. */ - rc = lsmFsWriteLog(pDb->pFS, pLog->iOff, &pLog->buf); - pLog->iOff += pLog->buf.n; - pLog->iCksumBuf = pLog->buf.n = 0; - - return rc; -} - -/* -** Write the contents of the log-buffer to disk. Then write either a CKSUM -** or COMMIT record, depending on the value of parameter eType. -*/ -static int logFlush(lsm_db *pDb, int eType){ - int rc; - int nReq; - LogWriter *pLog = pDb->pLogWriter; - - assert( eType==LSM_LOG_COMMIT ); - assert( pLog ); - - /* Commit record is always 9 bytes in size. */ - nReq = 9; - if( eType==LSM_LOG_COMMIT && pLog->szSector>1 ) nReq += pLog->szSector + 17; - rc = jumpIfRequired(pDb, pLog, nReq, 0); - - /* If this is a COMMIT, add padding to the log so that the COMMIT record - ** is aligned against the end of a disk sector. In other words, add padding - ** so that the first byte following the COMMIT record lies on a different - ** sector. */ - if( eType==LSM_LOG_COMMIT && pLog->szSector>1 ){ - int nPad; /* Bytes of padding to add */ - - /* Determine the value of nPad. */ - nPad = ((pLog->iOff + pLog->buf.n + 9) % pLog->szSector); - if( nPad ) nPad = pLog->szSector - nPad; - rc = lsmStringExtend(&pLog->buf, nPad); - if( rc!=LSM_OK ) return rc; - - while( nPad ){ - if( nPad==1 ){ - pLog->buf.z[pLog->buf.n++] = LSM_LOG_PAD1; - nPad = 0; - }else{ - int n = LSM_MIN(200, nPad-2); - pLog->buf.z[pLog->buf.n++] = LSM_LOG_PAD2; - pLog->buf.z[pLog->buf.n++] = (char)n; - nPad -= 2; - memset(&pLog->buf.z[pLog->buf.n], 0x2B, n); - pLog->buf.n += n; - nPad -= n; - } - } - } - - /* Make sure there is room in the log-buffer to add the CKSUM or COMMIT - ** record. Then add the first byte of it. */ - rc = lsmStringExtend(&pLog->buf, 9); - if( rc!=LSM_OK ) return rc; - pLog->buf.z[pLog->buf.n++] = (char)eType; - memset(&pLog->buf.z[pLog->buf.n], 0, 8); - - rc = logCksumAndFlush(pDb); - - /* If this is a commit and synchronous=full, sync the log to disk. */ - if( rc==LSM_OK && eType==LSM_LOG_COMMIT && pDb->eSafety==LSM_SAFETY_FULL ){ - rc = lsmFsSyncLog(pDb->pFS); - } - return rc; -} - -/* -** Append an LSM_LOG_WRITE (if nVal>=0) or LSM_LOG_DELETE (if nVal<0) -** record to the database log. -*/ -int lsmLogWrite( - lsm_db *pDb, /* Database handle */ - int eType, - void *pKey, int nKey, /* Database key to write to log */ - void *pVal, int nVal /* Database value (or nVal<0) to write */ -){ - int rc = LSM_OK; - LogWriter *pLog; /* Log object to write to */ - int nReq; /* Bytes of space required in log */ - int bCksum = 0; /* True to embed a checksum in this record */ - - assert( eType==LSM_WRITE || eType==LSM_DELETE || eType==LSM_DRANGE ); - assert( LSM_LOG_WRITE==LSM_WRITE ); - assert( LSM_LOG_DELETE==LSM_DELETE ); - assert( LSM_LOG_DRANGE==LSM_DRANGE ); - assert( (eType==LSM_LOG_DELETE)==(nVal<0) ); - - if( pDb->bUseLog==0 ) return LSM_OK; - pLog = pDb->pLogWriter; - - /* Determine how many bytes of space are required, assuming that a checksum - ** will be embedded in this record (even though it may not be). */ - nReq = 1 + lsmVarintLen32(nKey) + 8 + nKey; - if( eType!=LSM_LOG_DELETE ) nReq += lsmVarintLen32(nVal) + nVal; - - /* Jump over the jump region if required. Set bCksum to true to tell the - ** code below to include a checksum in the record if either (a) writing - ** this record would mean that more than LSM_CKSUM_MAXDATA bytes of data - ** have been written to the log since the last checksum, or (b) the jump - ** is taken. */ - rc = jumpIfRequired(pDb, pLog, nReq, &bCksum); - if( (pLog->buf.n+nReq) > LSM_CKSUM_MAXDATA ) bCksum = 1; - - if( rc==LSM_OK ){ - rc = lsmStringExtend(&pLog->buf, nReq); - } - if( rc==LSM_OK ){ - u8 *a = (u8 *)&pLog->buf.z[pLog->buf.n]; - - /* Write the record header - the type byte followed by either 1 (for - ** DELETE) or 2 (for WRITE) varints. */ - assert( LSM_LOG_WRITE_CKSUM == (LSM_LOG_WRITE | 0x0001) ); - assert( LSM_LOG_DELETE_CKSUM == (LSM_LOG_DELETE | 0x0001) ); - assert( LSM_LOG_DRANGE_CKSUM == (LSM_LOG_DRANGE | 0x0001) ); - *(a++) = (u8)eType | (u8)bCksum; - a += lsmVarintPut32(a, nKey); - if( eType!=LSM_LOG_DELETE ) a += lsmVarintPut32(a, nVal); - - if( bCksum ){ - pLog->buf.n = (a - (u8 *)pLog->buf.z); - rc = logCksumAndFlush(pDb); - a = (u8 *)&pLog->buf.z[pLog->buf.n]; - } - - memcpy(a, pKey, nKey); - a += nKey; - if( eType!=LSM_LOG_DELETE ){ - memcpy(a, pVal, nVal); - a += nVal; - } - pLog->buf.n = a - (u8 *)pLog->buf.z; - assert( pLog->buf.n<=pLog->buf.nAlloc ); - } - - return rc; -} - -/* -** Append an LSM_LOG_COMMIT record to the database log. -*/ -int lsmLogCommit(lsm_db *pDb){ - if( pDb->bUseLog==0 ) return LSM_OK; - return logFlush(pDb, LSM_LOG_COMMIT); -} - -/* -** Store the current offset and other checksum related information in the -** structure *pMark. Later, *pMark can be passed to lsmLogSeek() to "rewind" -** the LogWriter object to the current log file offset. This is used when -** rolling back savepoint transactions. -*/ -void lsmLogTell( - lsm_db *pDb, /* Database handle */ - LogMark *pMark /* Populate this object with current offset */ -){ - LogWriter *pLog; - int nCksum; - - if( pDb->bUseLog==0 ) return; - pLog = pDb->pLogWriter; - nCksum = pLog->buf.n & 0xFFFFFFF8; - logUpdateCksum(pLog, nCksum); - assert( pLog->iCksumBuf==nCksum ); - pMark->nBuf = pLog->buf.n - nCksum; - memcpy(pMark->aBuf, &pLog->buf.z[nCksum], pMark->nBuf); - - pMark->iOff = pLog->iOff + pLog->buf.n; - pMark->cksum0 = pLog->cksum0; - pMark->cksum1 = pLog->cksum1; -} - -/* -** Seek (rewind) back to the log file offset stored by an earlier call to -** lsmLogTell() in *pMark. -*/ -void lsmLogSeek( - lsm_db *pDb, /* Database handle */ - LogMark *pMark /* Object containing log offset to seek to */ -){ - LogWriter *pLog; - - if( pDb->bUseLog==0 ) return; - pLog = pDb->pLogWriter; - - assert( pMark->iOff<=pLog->iOff+pLog->buf.n ); - if( (pMark->iOff & 0xFFFFFFF8)>=pLog->iOff ){ - pLog->buf.n = (int)(pMark->iOff - pLog->iOff); - pLog->iCksumBuf = (pLog->buf.n & 0xFFFFFFF8); - }else{ - pLog->buf.n = pMark->nBuf; - memcpy(pLog->buf.z, pMark->aBuf, pMark->nBuf); - pLog->iCksumBuf = 0; - pLog->iOff = pMark->iOff - pMark->nBuf; - } - pLog->cksum0 = pMark->cksum0; - pLog->cksum1 = pMark->cksum1; - - if( pMark->iOff > pLog->iRegion1End ) pLog->iRegion1End = 0; - if( pMark->iOff > pLog->iRegion2Start ) pLog->iRegion2Start = 0; -} - -/* -** This function does the work for an lsm_info(LOG_STRUCTURE) request. -*/ -int lsmInfoLogStructure(lsm_db *pDb, char **pzVal){ - int rc = LSM_OK; - char *zVal = 0; - - /* If there is no read or write transaction open, read the latest - ** tree-header from shared-memory to report on. If necessary, update - ** it based on the contents of the database header. - ** - ** No locks are taken here - these are passive read operations only. - */ - if( pDb->pCsr==0 && pDb->nTransOpen==0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - if( rc==LSM_OK ) rc = logReclaimSpace(pDb); - } - - if( rc==LSM_OK ){ - DbLog *pLog = &pDb->treehdr.log; - zVal = lsmMallocPrintf(pDb->pEnv, - "%d %d %d %d %d %d", - (int)pLog->aRegion[0].iStart, (int)pLog->aRegion[0].iEnd, - (int)pLog->aRegion[1].iStart, (int)pLog->aRegion[1].iEnd, - (int)pLog->aRegion[2].iStart, (int)pLog->aRegion[2].iEnd - ); - if( !zVal ) rc = LSM_NOMEM_BKPT; - } - - *pzVal = zVal; - return rc; -} - -/************************************************************************* -** Begin code for log recovery. -*/ - -typedef struct LogReader LogReader; -struct LogReader { - FileSystem *pFS; /* File system to read from */ - i64 iOff; /* File offset at end of buf content */ - int iBuf; /* Current read offset in buf */ - LsmString buf; /* Buffer containing file content */ - - int iCksumBuf; /* Offset in buf corresponding to cksum[01] */ - u32 cksum0; /* Checksum 0 at offset iCksumBuf */ - u32 cksum1; /* Checksum 1 at offset iCksumBuf */ -}; - -static void logReaderBlob( - LogReader *p, /* Log reader object */ - LsmString *pBuf, /* Dynamic storage, if required */ - int nBlob, /* Number of bytes to read */ - u8 **ppBlob, /* OUT: Pointer to blob read */ - int *pRc /* IN/OUT: Error code */ -){ - static const int LOG_READ_SIZE = 512; - int rc = *pRc; /* Return code */ - int nReq = nBlob; /* Bytes required */ - - while( rc==LSM_OK && nReq>0 ){ - int nAvail; /* Bytes of data available in p->buf */ - if( p->buf.n==p->iBuf ){ - int nCksum; /* Total bytes requiring checksum */ - int nCarry = 0; /* Total bytes requiring checksum */ - - nCksum = p->iBuf - p->iCksumBuf; - if( nCksum>0 ){ - nCarry = nCksum % 8; - nCksum = ((nCksum / 8) * 8); - if( nCksum>0 ){ - logCksumUnaligned( - &p->buf.z[p->iCksumBuf], nCksum, &p->cksum0, &p->cksum1 - ); - } - } - if( nCarry>0 ) memcpy(p->buf.z, &p->buf.z[p->iBuf-nCarry], nCarry); - p->buf.n = nCarry; - p->iBuf = nCarry; - - rc = lsmFsReadLog(p->pFS, p->iOff, LOG_READ_SIZE, &p->buf); - if( rc!=LSM_OK ) break; - p->iCksumBuf = 0; - p->iOff += LOG_READ_SIZE; - } - - nAvail = p->buf.n - p->iBuf; - if( ppBlob && nReq==nBlob && nBlob<=nAvail ){ - *ppBlob = (u8 *)&p->buf.z[p->iBuf]; - p->iBuf += nBlob; - nReq = 0; - }else{ - int nCopy = LSM_MIN(nAvail, nReq); - if( nBlob==nReq ){ - pBuf->n = 0; - } - rc = lsmStringBinAppend(pBuf, (u8 *)&p->buf.z[p->iBuf], nCopy); - nReq -= nCopy; - p->iBuf += nCopy; - if( nReq==0 && ppBlob ){ - *ppBlob = (u8*)pBuf->z; - } - } - } - - *pRc = rc; -} - -static void logReaderVarint( - LogReader *p, - LsmString *pBuf, - int *piVal, /* OUT: Value read from log */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==LSM_OK ){ - u8 *aVarint; - if( p->buf.n==p->iBuf ){ - logReaderBlob(p, 0, 10, &aVarint, pRc); - if( LSM_OK==*pRc ) p->iBuf -= (10 - lsmVarintGet32(aVarint, piVal)); - }else{ - logReaderBlob(p, pBuf, lsmVarintSize(p->buf.z[p->iBuf]), &aVarint, pRc); - if( LSM_OK==*pRc ) lsmVarintGet32(aVarint, piVal); - } - } -} - -static void logReaderByte(LogReader *p, u8 *pByte, int *pRc){ - u8 *pPtr = 0; - logReaderBlob(p, 0, 1, &pPtr, pRc); - if( pPtr ) *pByte = *pPtr; -} - -static void logReaderCksum(LogReader *p, LsmString *pBuf, int *pbEof, int *pRc){ - if( *pRc==LSM_OK ){ - u8 *pPtr = 0; - u32 cksum0, cksum1; - int nCksum = p->iBuf - p->iCksumBuf; - - /* Update in-memory (expected) checksums */ - assert( nCksum>=0 ); - logCksumUnaligned(&p->buf.z[p->iCksumBuf], nCksum, &p->cksum0, &p->cksum1); - p->iCksumBuf = p->iBuf + 8; - logReaderBlob(p, pBuf, 8, &pPtr, pRc); - assert( pPtr || *pRc ); - - /* Read the checksums from the log file. Set *pbEof if they do not match. */ - if( pPtr ){ - cksum0 = lsmGetU32(pPtr); - cksum1 = lsmGetU32(&pPtr[4]); - *pbEof = (cksum0!=p->cksum0 || cksum1!=p->cksum1); - p->iCksumBuf = p->iBuf; - } - } -} - -static void logReaderInit( - lsm_db *pDb, /* Database handle */ - DbLog *pLog, /* Log object associated with pDb */ - int bInitBuf, /* True if p->buf is uninitialized */ - LogReader *p /* Initialize this LogReader object */ -){ - p->pFS = pDb->pFS; - p->iOff = pLog->aRegion[2].iStart; - p->cksum0 = pLog->cksum0; - p->cksum1 = pLog->cksum1; - if( bInitBuf ){ lsmStringInit(&p->buf, pDb->pEnv); } - p->buf.n = 0; - p->iCksumBuf = 0; - p->iBuf = 0; -} - -/* -** This function is called after reading the header of a LOG_DELETE or -** LOG_WRITE record. Parameter nByte is the total size of the key and -** value that follow the header just read. Return true if the size and -** position of the record indicate that it should contain a checksum. -*/ -static int logRequireCksum(LogReader *p, int nByte){ - return ((p->iBuf + nByte - p->iCksumBuf) > LSM_CKSUM_MAXDATA); -} - -/* -** Recover the contents of the log file. -*/ -int lsmLogRecover(lsm_db *pDb){ - LsmString buf1; /* Key buffer */ - LsmString buf2; /* Value buffer */ - LogReader reader; /* Log reader object */ - int rc = LSM_OK; /* Return code */ - int nCommit = 0; /* Number of transactions to recover */ - int iPass; - int nJump = 0; /* Number of LSM_LOG_JUMP records in pass 0 */ - DbLog *pLog; - int bOpen; - - rc = lsmFsOpenLog(pDb, &bOpen); - if( rc!=LSM_OK ) return rc; - - rc = lsmTreeInit(pDb); - if( rc!=LSM_OK ) return rc; - - pLog = &pDb->treehdr.log; - lsmCheckpointLogoffset(pDb->pShmhdr->aSnap2, pLog); - - logReaderInit(pDb, pLog, 1, &reader); - lsmStringInit(&buf1, pDb->pEnv); - lsmStringInit(&buf2, pDb->pEnv); - - /* The outer for() loop runs at most twice. The first iteration is to - ** count the number of committed transactions in the log. The second - ** iterates through those transactions and updates the in-memory tree - ** structure with their contents. */ - if( bOpen ){ - for(iPass=0; iPass<2 && rc==LSM_OK; iPass++){ - int bEof = 0; - - while( rc==LSM_OK && !bEof ){ - u8 eType = 0; - logReaderByte(&reader, &eType, &rc); - - switch( eType ){ - case LSM_LOG_PAD1: - break; - - case LSM_LOG_PAD2: { - int nPad; - logReaderVarint(&reader, &buf1, &nPad, &rc); - logReaderBlob(&reader, &buf1, nPad, 0, &rc); - break; - } - - case LSM_LOG_DRANGE: - case LSM_LOG_DRANGE_CKSUM: - case LSM_LOG_WRITE: - case LSM_LOG_WRITE_CKSUM: { - int nKey; - int nVal; - u8 *aVal; - logReaderVarint(&reader, &buf1, &nKey, &rc); - logReaderVarint(&reader, &buf2, &nVal, &rc); - - if( eType==LSM_LOG_WRITE_CKSUM || eType==LSM_LOG_DRANGE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey+nVal); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, 0, &rc); - logReaderBlob(&reader, &buf2, nVal, &aVal, &rc); - if( iPass==1 && rc==LSM_OK ){ - if( eType==LSM_LOG_WRITE || eType==LSM_LOG_WRITE_CKSUM ){ - rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal); - }else{ - rc = lsmTreeDelete(pDb, (u8 *)buf1.z, nKey, aVal, nVal); - } - } - break; - } - - case LSM_LOG_DELETE: - case LSM_LOG_DELETE_CKSUM: { - int nKey; u8 *aKey; - logReaderVarint(&reader, &buf1, &nKey, &rc); - - if( eType==LSM_LOG_DELETE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, &aKey, &rc); - if( iPass==1 && rc==LSM_OK ){ - rc = lsmTreeInsert(pDb, aKey, nKey, NULL, -1); - } - break; - } - - case LSM_LOG_COMMIT: - logReaderCksum(&reader, &buf1, &bEof, &rc); - if( bEof==0 ){ - nCommit++; - assert( nCommit>0 || iPass==1 ); - if( nCommit==0 ) bEof = 1; - } - break; - - case LSM_LOG_JUMP: { - int iOff = 0; - logReaderVarint(&reader, &buf1, &iOff, &rc); - if( rc==LSM_OK ){ - if( iPass==1 ){ - if( pLog->aRegion[2].iStart==0 ){ - assert( pLog->aRegion[1].iStart==0 ); - pLog->aRegion[1].iEnd = reader.iOff; - }else{ - assert( pLog->aRegion[0].iStart==0 ); - pLog->aRegion[0].iStart = pLog->aRegion[2].iStart; - pLog->aRegion[0].iEnd = reader.iOff-reader.buf.n+reader.iBuf; - } - pLog->aRegion[2].iStart = iOff; - }else{ - if( (nJump++)==2 ){ - bEof = 1; - } - } - - reader.iOff = iOff; - reader.buf.n = reader.iBuf; - } - break; - } - - default: - /* Including LSM_LOG_EOF */ - bEof = 1; - break; - } - } - - if( rc==LSM_OK && iPass==0 ){ - if( nCommit==0 ){ - if( pLog->aRegion[2].iStart==0 ){ - iPass = 1; - }else{ - pLog->aRegion[2].iStart = 0; - iPass = -1; - lsmCheckpointZeroLogoffset(pDb); - } - } - logReaderInit(pDb, pLog, 0, &reader); - nCommit = nCommit * -1; - } - } - } - - /* Initialize DbLog object */ - if( rc==LSM_OK ){ - pLog->aRegion[2].iEnd = reader.iOff - reader.buf.n + reader.iBuf; - pLog->cksum0 = reader.cksum0; - pLog->cksum1 = reader.cksum1; - } - - if( rc==LSM_OK ){ - rc = lsmFinishRecovery(pDb); - }else{ - lsmFinishRecovery(pDb); - } - - if( pDb->bRoTrans ){ - lsmFsCloseLog(pDb); - } - - lsmStringClear(&buf1); - lsmStringClear(&buf2); - lsmStringClear(&reader.buf); - return rc; -} - -void lsmLogClose(lsm_db *db){ - if( db->pLogWriter ){ - lsmFree(db->pEnv, db->pLogWriter->buf.z); - lsmFree(db->pEnv, db->pLogWriter); - db->pLogWriter = 0; - } -} diff --git a/ext/lsm1/lsm_main.c b/ext/lsm1/lsm_main.c deleted file mode 100644 index f2b353105..000000000 --- a/ext/lsm1/lsm_main.c +++ /dev/null @@ -1,1008 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** The main interface to the LSM module. -*/ -#include "lsmInt.h" - - -#ifdef LSM_DEBUG -/* -** This function returns a copy of its only argument. -** -** When the library is built with LSM_DEBUG defined, this function is called -** whenever an error code is generated (not propagated - generated). So -** if the library is mysteriously returning (say) LSM_IOERR, a breakpoint -** may be set in this function to determine why. -*/ -int lsmErrorBkpt(int rc){ - /* Set breakpoint here! */ - return rc; -} - -/* -** This function contains various assert() statements that test that the -** lsm_db structure passed as an argument is internally consistent. -*/ -static void assert_db_state(lsm_db *pDb){ - - /* If there is at least one cursor or a write transaction open, the database - ** handle must be holding a pointer to a client snapshot. And the reverse - ** - if there are no open cursors and no write transactions then there must - ** not be a client snapshot. */ - - assert( (pDb->pCsr!=0||pDb->nTransOpen>0)==(pDb->iReader>=0||pDb->bRoTrans) ); - - assert( (pDb->iReader<0 && pDb->bRoTrans==0) || pDb->pClient!=0 ); - - assert( pDb->nTransOpen>=0 ); -} -#else -# define assert_db_state(x) -#endif - -/* -** The default key-compare function. -*/ -static int xCmp(void *p1, int n1, void *p2, int n2){ - int res; - res = memcmp(p1, p2, LSM_MIN(n1, n2)); - if( res==0 ) res = (n1-n2); - return res; -} - -static void xLog(void *pCtx, int rc, const char *z){ - (void)(rc); - (void)(pCtx); - fprintf(stderr, "%s\n", z); - fflush(stderr); -} - -/* -** Allocate a new db handle. -*/ -int lsm_new(lsm_env *pEnv, lsm_db **ppDb){ - lsm_db *pDb; - - /* If the user did not provide an environment, use the default. */ - if( pEnv==0 ) pEnv = lsm_default_env(); - assert( pEnv ); - - /* Allocate the new database handle */ - *ppDb = pDb = (lsm_db *)lsmMallocZero(pEnv, sizeof(lsm_db)); - if( pDb==0 ) return LSM_NOMEM_BKPT; - - /* Initialize the new object */ - pDb->pEnv = pEnv; - pDb->nTreeLimit = LSM_DFLT_AUTOFLUSH; - pDb->nAutockpt = LSM_DFLT_AUTOCHECKPOINT; - pDb->bAutowork = LSM_DFLT_AUTOWORK; - pDb->eSafety = LSM_DFLT_SAFETY; - pDb->xCmp = xCmp; - pDb->nDfltPgsz = LSM_DFLT_PAGE_SIZE; - pDb->nDfltBlksz = LSM_DFLT_BLOCK_SIZE; - pDb->nMerge = LSM_DFLT_AUTOMERGE; - pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES; - pDb->bUseLog = LSM_DFLT_USE_LOG; - pDb->iReader = -1; - pDb->iRwclient = -1; - pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; - pDb->iMmap = LSM_DFLT_MMAP; - pDb->xLog = xLog; - pDb->compress.iId = LSM_COMPRESSION_NONE; - return LSM_OK; -} - -lsm_env *lsm_get_env(lsm_db *pDb){ - assert( pDb->pEnv ); - return pDb->pEnv; -} - -/* -** If database handle pDb is currently holding a client snapshot, but does -** not have any open cursors or write transactions, release it. -*/ -static void dbReleaseClientSnapshot(lsm_db *pDb){ - if( pDb->nTransOpen==0 && pDb->pCsr==0 ){ - lsmFinishReadTrans(pDb); - } -} - -static int getFullpathname( - lsm_env *pEnv, - const char *zRel, - char **pzAbs -){ - int nAlloc = 0; - char *zAlloc = 0; - int nReq = 0; - int rc; - - do{ - nAlloc = nReq; - rc = pEnv->xFullpath(pEnv, zRel, zAlloc, &nReq); - if( nReq>nAlloc ){ - zAlloc = lsmReallocOrFreeRc(pEnv, zAlloc, nReq, &rc); - } - }while( nReq>nAlloc && rc==LSM_OK ); - - if( rc!=LSM_OK ){ - lsmFree(pEnv, zAlloc); - zAlloc = 0; - } - *pzAbs = zAlloc; - return rc; -} - -/* -** Check that the bits in the db->mLock mask are consistent with the -** value stored in db->iRwclient. An assert shall fail otherwise. -*/ -static void assertRwclientLockValue(lsm_db *db){ -#ifndef NDEBUG - u64 msk; /* Mask of mLock bits for RWCLIENT locks */ - u64 rwclient = 0; /* Bit corresponding to db->iRwclient */ - - if( db->iRwclient>=0 ){ - rwclient = ((u64)1 << (LSM_LOCK_RWCLIENT(db->iRwclient)-1)); - } - msk = ((u64)1 << (LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT)-1)) - 1; - msk -= (((u64)1 << (LSM_LOCK_RWCLIENT(0)-1)) - 1); - - assert( (db->mLock & msk)==rwclient ); -#endif -} - -/* -** Open a new connection to database zFilename. -*/ -int lsm_open(lsm_db *pDb, const char *zFilename){ - int rc; - - if( pDb->pDatabase ){ - rc = LSM_MISUSE; - }else{ - char *zFull; - - /* Translate the possibly relative pathname supplied by the user into - ** an absolute pathname. This is required because the supplied path - ** is used (either directly or with "-log" appended to it) for more - ** than one purpose - to open both the database and log files, and - ** perhaps to unlink the log file during disconnection. An absolute - ** path is required to ensure that the correct files are operated - ** on even if the application changes the cwd. */ - rc = getFullpathname(pDb->pEnv, zFilename, &zFull); - assert( rc==LSM_OK || zFull==0 ); - - /* Connect to the database. */ - if( rc==LSM_OK ){ - rc = lsmDbDatabaseConnect(pDb, zFull); - } - - if( pDb->bReadonly==0 ){ - /* Configure the file-system connection with the page-size and block-size - ** of this database. Even if the database file is zero bytes in size - ** on disk, these values have been set in shared-memory by now, and so - ** are guaranteed not to change during the lifetime of this connection. - */ - if( rc==LSM_OK && LSM_OK==(rc = lsmCheckpointLoad(pDb, 0)) ){ - lsmFsSetPageSize(pDb->pFS, lsmCheckpointPgsz(pDb->aSnapshot)); - lsmFsSetBlockSize(pDb->pFS, lsmCheckpointBlksz(pDb->aSnapshot)); - } - } - - lsmFree(pDb->pEnv, zFull); - assertRwclientLockValue(pDb); - } - - assert( pDb->bReadonly==0 || pDb->bReadonly==1 ); - assert( rc!=LSM_OK || (pDb->pShmhdr==0)==(pDb->bReadonly==1) ); - - return rc; -} - -int lsm_close(lsm_db *pDb){ - int rc = LSM_OK; - if( pDb ){ - assert_db_state(pDb); - if( pDb->pCsr || pDb->nTransOpen ){ - rc = LSM_MISUSE_BKPT; - }else{ - lsmMCursorFreeCache(pDb); - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - - assertRwclientLockValue(pDb); - - lsmDbDatabaseRelease(pDb); - lsmLogClose(pDb); - lsmFsClose(pDb->pFS); - /* assert( pDb->mLock==0 ); */ - - /* Invoke any destructors registered for the compression or - ** compression factory callbacks. */ - if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); - if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); - - lsmFree(pDb->pEnv, pDb->rollback.aArray); - lsmFree(pDb->pEnv, pDb->aTrans); - lsmFree(pDb->pEnv, pDb->apShm); - lsmFree(pDb->pEnv, pDb); - } - } - return rc; -} - -int lsm_config(lsm_db *pDb, int eParam, ...){ - int rc = LSM_OK; - va_list ap; - va_start(ap, eParam); - - switch( eParam ){ - case LSM_CONFIG_AUTOFLUSH: { - /* This parameter is read and written in KB. But all internal - ** processing is done in bytes. */ - int *piVal = va_arg(ap, int *); - int iVal = *piVal; - if( iVal>=0 && iVal<=(1024*1024) ){ - pDb->nTreeLimit = iVal*1024; - } - *piVal = (pDb->nTreeLimit / 1024); - break; - } - - case LSM_CONFIG_AUTOWORK: { - int *piVal = va_arg(ap, int *); - if( *piVal>=0 ){ - pDb->bAutowork = *piVal; - } - *piVal = pDb->bAutowork; - break; - } - - case LSM_CONFIG_AUTOCHECKPOINT: { - /* This parameter is read and written in KB. But all internal processing - ** (including the lsm_db.nAutockpt variable) is done in bytes. */ - int *piVal = va_arg(ap, int *); - if( *piVal>=0 ){ - int iVal = *piVal; - pDb->nAutockpt = (i64)iVal * 1024; - } - *piVal = (int)(pDb->nAutockpt / 1024); - break; - } - - case LSM_CONFIG_PAGE_SIZE: { - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the page-size according to the - ** FileSystem object. */ - *piVal = lsmFsPageSize(pDb->pFS); - }else{ - if( *piVal>=256 && *piVal<=65536 && ((*piVal-1) & *piVal)==0 ){ - pDb->nDfltPgsz = *piVal; - }else{ - *piVal = pDb->nDfltPgsz; - } - } - break; - } - - case LSM_CONFIG_BLOCK_SIZE: { - /* This parameter is read and written in KB. But all internal - ** processing is done in bytes. */ - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the block-size in KB according to the - ** FileSystem object. */ - *piVal = lsmFsBlockSize(pDb->pFS) / 1024; - }else{ - int iVal = *piVal; - if( iVal>=64 && iVal<=65536 && ((iVal-1) & iVal)==0 ){ - pDb->nDfltBlksz = iVal * 1024; - }else{ - *piVal = pDb->nDfltBlksz / 1024; - } - } - break; - } - - case LSM_CONFIG_SAFETY: { - int *piVal = va_arg(ap, int *); - if( *piVal>=0 && *piVal<=2 ){ - pDb->eSafety = *piVal; - } - *piVal = pDb->eSafety; - break; - } - - case LSM_CONFIG_MMAP: { - int *piVal = va_arg(ap, int *); - if( pDb->iReader<0 && *piVal>=0 ){ - pDb->iMmap = *piVal; - rc = lsmFsConfigure(pDb); - } - *piVal = pDb->iMmap; - break; - } - - case LSM_CONFIG_USE_LOG: { - int *piVal = va_arg(ap, int *); - if( pDb->nTransOpen==0 && (*piVal==0 || *piVal==1) ){ - pDb->bUseLog = *piVal; - } - *piVal = pDb->bUseLog; - break; - } - - case LSM_CONFIG_AUTOMERGE: { - int *piVal = va_arg(ap, int *); - if( *piVal>1 ) pDb->nMerge = *piVal; - *piVal = pDb->nMerge; - break; - } - - case LSM_CONFIG_MAX_FREELIST: { - int *piVal = va_arg(ap, int *); - if( *piVal>=2 && *piVal<=LSM_MAX_FREELIST_ENTRIES ){ - pDb->nMaxFreelist = *piVal; - } - *piVal = pDb->nMaxFreelist; - break; - } - - case LSM_CONFIG_MULTIPLE_PROCESSES: { - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to true if this connection is currently - ** in multi-process mode. */ - *piVal = lsmDbMultiProc(pDb); - }else{ - pDb->bMultiProc = *piVal = (*piVal!=0); - } - break; - } - - case LSM_CONFIG_READONLY: { - int *piVal = va_arg(ap, int *); - /* If lsm_open() has been called, this is a read-only parameter. */ - if( pDb->pDatabase==0 && *piVal>=0 ){ - pDb->bReadonly = *piVal = (*piVal!=0); - } - *piVal = pDb->bReadonly; - break; - } - - case LSM_CONFIG_SET_COMPRESSION: { - lsm_compress *p = va_arg(ap, lsm_compress *); - if( pDb->iReader>=0 && pDb->bInFactory==0 ){ - /* May not change compression schemes with an open transaction */ - rc = LSM_MISUSE_BKPT; - }else{ - if( pDb->compress.xFree ){ - /* Invoke any destructor belonging to the current compression. */ - pDb->compress.xFree(pDb->compress.pCtx); - } - if( p->xBound==0 ){ - memset(&pDb->compress, 0, sizeof(lsm_compress)); - pDb->compress.iId = LSM_COMPRESSION_NONE; - }else{ - memcpy(&pDb->compress, p, sizeof(lsm_compress)); - } - rc = lsmFsConfigure(pDb); - } - break; - } - - case LSM_CONFIG_SET_COMPRESSION_FACTORY: { - lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); - if( pDb->factory.xFree ){ - /* Invoke any destructor belonging to the current factory. */ - pDb->factory.xFree(pDb->factory.pCtx); - } - memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); - break; - } - - case LSM_CONFIG_GET_COMPRESSION: { - lsm_compress *p = va_arg(ap, lsm_compress *); - memcpy(p, &pDb->compress, sizeof(lsm_compress)); - break; - } - - default: - rc = LSM_MISUSE; - break; - } - - va_end(ap); - return rc; -} - -void lsmAppendSegmentList(LsmString *pStr, char *zPre, Segment *pSeg){ - lsmStringAppendf(pStr, "%s{%lld %lld %lld %lld}", zPre, - pSeg->iFirst, pSeg->iLastPg, pSeg->iRoot, pSeg->nSize - ); -} - -static int infoGetWorker(lsm_db *pDb, Snapshot **pp, int *pbUnlock){ - int rc = LSM_OK; - - assert( *pbUnlock==0 ); - if( !pDb->pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - *pbUnlock = 1; - } - if( pp ) *pp = pDb->pWorker; - return rc; -} - -static void infoFreeWorker(lsm_db *pDb, int bUnlock){ - if( bUnlock ){ - int rcdummy = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcdummy); - } -} - -int lsmStructList( - lsm_db *pDb, /* Database handle */ - char **pzOut /* OUT: Nul-terminated string (tcl list) */ -){ - Level *pTopLevel = 0; /* Top level of snapshot to report on */ - int rc = LSM_OK; - Level *p; - LsmString s; - Snapshot *pWorker; /* Worker snapshot */ - int bUnlock = 0; - - /* Obtain the worker snapshot */ - rc = infoGetWorker(pDb, &pWorker, &bUnlock); - if( rc!=LSM_OK ) return rc; - - /* Format the contents of the snapshot as text */ - pTopLevel = lsmDbSnapshotLevel(pWorker); - lsmStringInit(&s, pDb->pEnv); - for(p=pTopLevel; rc==LSM_OK && p; p=p->pNext){ - int i; - lsmStringAppendf(&s, "%s{%d", (s.n ? " " : ""), (int)p->iAge); - lsmAppendSegmentList(&s, " ", &p->lhs); - for(i=0; rc==LSM_OK && i<p->nRight; i++){ - lsmAppendSegmentList(&s, " ", &p->aRhs[i]); - } - lsmStringAppend(&s, "}", 1); - } - rc = s.n>=0 ? LSM_OK : LSM_NOMEM; - - /* Release the snapshot and return */ - infoFreeWorker(pDb, bUnlock); - *pzOut = s.z; - return rc; -} - -static int infoFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - LsmString *pStr = (LsmString *)pCtx; - lsmStringAppendf(pStr, "%s{%d %lld}", (pStr->n?" ":""), iBlk, iSnapshot); - return 0; -} - -int lsmInfoFreelist(lsm_db *pDb, char **pzOut){ - Snapshot *pWorker; /* Worker snapshot */ - int bUnlock = 0; - LsmString s; - int rc; - - /* Obtain the worker snapshot */ - rc = infoGetWorker(pDb, &pWorker, &bUnlock); - if( rc!=LSM_OK ) return rc; - - lsmStringInit(&s, pDb->pEnv); - rc = lsmWalkFreelist(pDb, 0, infoFreelistCb, &s); - if( rc!=LSM_OK ){ - lsmFree(pDb->pEnv, s.z); - }else{ - *pzOut = s.z; - } - - /* Release the snapshot and return */ - infoFreeWorker(pDb, bUnlock); - return rc; -} - -static int infoTreeSize(lsm_db *db, int *pnOldKB, int *pnNewKB){ - ShmHeader *pShm = db->pShmhdr; - TreeHeader *p = &pShm->hdr1; - - /* The following code suffers from two race conditions, as it accesses and - ** trusts the contents of shared memory without verifying checksums: - ** - ** * The two values read - TreeHeader.root.nByte and oldroot.nByte - are - ** 32-bit fields. It is assumed that reading from one of these - ** is atomic - that it is not possible to read a partially written - ** garbage value. However the two values may be mutually inconsistent. - ** - ** * TreeHeader.iLogOff is a 64-bit value. And lsmCheckpointLogOffset() - ** reads a 64-bit value from a snapshot stored in shared memory. It - ** is assumed that in each case it is possible to read a partially - ** written garbage value. If this occurs, then the value returned - ** for the size of the "old" tree may reflect the size of an "old" - ** tree that was recently flushed to disk. - ** - ** Given the context in which this function is called (as a result of an - ** lsm_info(LSM_INFO_TREE_SIZE) request), neither of these are considered to - ** be problems. - */ - *pnNewKB = ((int)p->root.nByte + 1023) / 1024; - if( p->iOldShmid ){ - if( p->iOldLog==lsmCheckpointLogOffset(pShm->aSnap1) ){ - *pnOldKB = 0; - }else{ - *pnOldKB = ((int)p->oldroot.nByte + 1023) / 1024; - } - }else{ - *pnOldKB = 0; - } - - return LSM_OK; -} - -int lsm_info(lsm_db *pDb, int eParam, ...){ - int rc = LSM_OK; - va_list ap; - va_start(ap, eParam); - - switch( eParam ){ - case LSM_INFO_NWRITE: { - int *piVal = va_arg(ap, int *); - *piVal = lsmFsNWrite(pDb->pFS); - break; - } - - case LSM_INFO_NREAD: { - int *piVal = va_arg(ap, int *); - *piVal = lsmFsNRead(pDb->pFS); - break; - } - - case LSM_INFO_DB_STRUCTURE: { - char **pzVal = va_arg(ap, char **); - rc = lsmStructList(pDb, pzVal); - break; - } - - case LSM_INFO_ARRAY_STRUCTURE: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal); - break; - } - - case LSM_INFO_ARRAY_PAGES: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayPages(pDb, pgno, pzVal); - break; - } - - case LSM_INFO_PAGE_HEX_DUMP: - case LSM_INFO_PAGE_ASCII_DUMP: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - int bUnlock = 0; - rc = infoGetWorker(pDb, 0, &bUnlock); - if( rc==LSM_OK ){ - int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP); - rc = lsmInfoPageDump(pDb, pgno, bHex, pzVal); - } - infoFreeWorker(pDb, bUnlock); - break; - } - - case LSM_INFO_LOG_STRUCTURE: { - char **pzVal = va_arg(ap, char **); - rc = lsmInfoLogStructure(pDb, pzVal); - break; - } - - case LSM_INFO_FREELIST: { - char **pzVal = va_arg(ap, char **); - rc = lsmInfoFreelist(pDb, pzVal); - break; - } - - case LSM_INFO_CHECKPOINT_SIZE: { - int *pnKB = va_arg(ap, int *); - rc = lsmCheckpointSize(pDb, pnKB); - break; - } - - case LSM_INFO_TREE_SIZE: { - int *pnOld = va_arg(ap, int *); - int *pnNew = va_arg(ap, int *); - rc = infoTreeSize(pDb, pnOld, pnNew); - break; - } - - case LSM_INFO_COMPRESSION_ID: { - unsigned int *piOut = va_arg(ap, unsigned int *); - if( pDb->pClient ){ - *piOut = pDb->pClient->iCmpId; - }else{ - rc = lsmInfoCompressionId(pDb, piOut); - } - break; - } - - default: - rc = LSM_MISUSE; - break; - } - - va_end(ap); - return rc; -} - -static int doWriteOp( - lsm_db *pDb, - int bDeleteRange, - const void *pKey, int nKey, /* Key to write or delete */ - const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ -){ - int rc = LSM_OK; /* Return code */ - int bCommit = 0; /* True to commit before returning */ - - if( pDb->nTransOpen==0 ){ - bCommit = 1; - rc = lsm_begin(pDb, 1); - } - - if( rc==LSM_OK ){ - int eType = (bDeleteRange ? LSM_DRANGE : (nVal>=0?LSM_WRITE:LSM_DELETE)); - rc = lsmLogWrite(pDb, eType, (void *)pKey, nKey, (void *)pVal, nVal); - } - - lsmSortedSaveTreeCursors(pDb); - - if( rc==LSM_OK ){ - int pgsz = lsmFsPageSize(pDb->pFS); - int nQuant = LSM_AUTOWORK_QUANT * pgsz; - int nBefore; - int nAfter; - int nDiff; - - if( nQuant>pDb->nTreeLimit ){ - nQuant = LSM_MAX(pDb->nTreeLimit, pgsz); - } - - nBefore = lsmTreeSize(pDb); - if( bDeleteRange ){ - rc = lsmTreeDelete(pDb, (void *)pKey, nKey, (void *)pVal, nVal); - }else{ - rc = lsmTreeInsert(pDb, (void *)pKey, nKey, (void *)pVal, nVal); - } - - nAfter = lsmTreeSize(pDb); - nDiff = (nAfter/nQuant) - (nBefore/nQuant); - if( rc==LSM_OK && pDb->bAutowork && nDiff!=0 ){ - rc = lsmSortedAutoWork(pDb, nDiff * LSM_AUTOWORK_QUANT); - } - } - - /* If a transaction was opened at the start of this function, commit it. - ** Or, if an error has occurred, roll it back. */ - if( bCommit ){ - if( rc==LSM_OK ){ - rc = lsm_commit(pDb, 0); - }else{ - lsm_rollback(pDb, 0); - } - } - - return rc; -} - -/* -** Write a new value into the database. -*/ -int lsm_insert( - lsm_db *db, /* Database connection */ - const void *pKey, int nKey, /* Key to write or delete */ - const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ -){ - return doWriteOp(db, 0, pKey, nKey, pVal, nVal); -} - -/* -** Delete a value from the database. -*/ -int lsm_delete(lsm_db *db, const void *pKey, int nKey){ - return doWriteOp(db, 0, pKey, nKey, 0, -1); -} - -/* -** Delete a range of database keys. -*/ -int lsm_delete_range( - lsm_db *db, /* Database handle */ - const void *pKey1, int nKey1, /* Lower bound of range to delete */ - const void *pKey2, int nKey2 /* Upper bound of range to delete */ -){ - int rc = LSM_OK; - if( db->xCmp((void *)pKey1, nKey1, (void *)pKey2, nKey2)<0 ){ - rc = doWriteOp(db, 1, pKey1, nKey1, pKey2, nKey2); - } - return rc; -} - -/* -** Open a new cursor handle. -** -** If there are currently no other open cursor handles, and no open write -** transaction, open a read transaction here. -*/ -int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr){ - int rc = LSM_OK; /* Return code */ - MultiCursor *pCsr = 0; /* New cursor object */ - - /* Open a read transaction if one is not already open. */ - assert_db_state(pDb); - - if( pDb->pShmhdr==0 ){ - assert( pDb->bReadonly ); - rc = lsmBeginRoTrans(pDb); - }else if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Allocate the multi-cursor. */ - if( rc==LSM_OK ){ - rc = lsmMCursorNew(pDb, &pCsr); - } - - /* If an error has occured, set the output to NULL and delete any partially - ** allocated cursor. If this means there are no open cursors, release the - ** client snapshot. */ - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - dbReleaseClientSnapshot(pDb); - } - - assert_db_state(pDb); - *ppCsr = (lsm_cursor *)pCsr; - return rc; -} - -/* -** Close a cursor opened using lsm_csr_open(). -*/ -int lsm_csr_close(lsm_cursor *p){ - if( p ){ - lsm_db *pDb = lsmMCursorDb((MultiCursor *)p); - assert_db_state(pDb); - lsmMCursorClose((MultiCursor *)p, 1); - dbReleaseClientSnapshot(pDb); - assert_db_state(pDb); - } - return LSM_OK; -} - -/* -** Attempt to seek the cursor to the database entry specified by pKey/nKey. -** If an error occurs (e.g. an OOM or IO error), return an LSM error code. -** Otherwise, return LSM_OK. -*/ -int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek){ - return lsmMCursorSeek((MultiCursor *)pCsr, 0, (void *)pKey, nKey, eSeek); -} - -int lsm_csr_next(lsm_cursor *pCsr){ - return lsmMCursorNext((MultiCursor *)pCsr); -} - -int lsm_csr_prev(lsm_cursor *pCsr){ - return lsmMCursorPrev((MultiCursor *)pCsr); -} - -int lsm_csr_first(lsm_cursor *pCsr){ - return lsmMCursorFirst((MultiCursor *)pCsr); -} - -int lsm_csr_last(lsm_cursor *pCsr){ - return lsmMCursorLast((MultiCursor *)pCsr); -} - -int lsm_csr_valid(lsm_cursor *pCsr){ - return lsmMCursorValid((MultiCursor *)pCsr); -} - -int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey){ - return lsmMCursorKey((MultiCursor *)pCsr, (void **)ppKey, pnKey); -} - -int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal){ - return lsmMCursorValue((MultiCursor *)pCsr, (void **)ppVal, pnVal); -} - -void lsm_config_log( - lsm_db *pDb, - void (*xLog)(void *, int, const char *), - void *pCtx -){ - pDb->xLog = xLog; - pDb->pLogCtx = pCtx; -} - -void lsm_config_work_hook( - lsm_db *pDb, - void (*xWork)(lsm_db *, void *), - void *pCtx -){ - pDb->xWork = xWork; - pDb->pWorkCtx = pCtx; -} - -void lsmLogMessage(lsm_db *pDb, int rc, const char *zFormat, ...){ - if( pDb->xLog ){ - LsmString s; - va_list ap, ap2; - lsmStringInit(&s, pDb->pEnv); - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(&s, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); - pDb->xLog(pDb->pLogCtx, rc, s.z); - lsmStringClear(&s); - } -} - -int lsm_begin(lsm_db *pDb, int iLevel){ - int rc; - - assert_db_state( pDb ); - rc = (pDb->bReadonly ? LSM_READONLY : LSM_OK); - - /* A value less than zero means open one more transaction. */ - if( iLevel<0 ) iLevel = pDb->nTransOpen + 1; - if( iLevel>pDb->nTransOpen ){ - int i; - - /* Extend the pDb->aTrans[] array if required. */ - if( rc==LSM_OK && pDb->nTransAlloc<iLevel ){ - TransMark *aNew; /* New allocation */ - int nByte = sizeof(TransMark) * (iLevel+1); - aNew = (TransMark *)lsmRealloc(pDb->pEnv, pDb->aTrans, nByte); - if( !aNew ){ - rc = LSM_NOMEM; - }else{ - nByte = sizeof(TransMark) * (iLevel+1 - pDb->nTransAlloc); - memset(&aNew[pDb->nTransAlloc], 0, nByte); - pDb->nTransAlloc = iLevel+1; - pDb->aTrans = aNew; - } - } - - if( rc==LSM_OK && pDb->nTransOpen==0 ){ - rc = lsmBeginWriteTrans(pDb); - } - - if( rc==LSM_OK ){ - for(i=pDb->nTransOpen; i<iLevel; i++){ - lsmTreeMark(pDb, &pDb->aTrans[i].tree); - lsmLogTell(pDb, &pDb->aTrans[i].log); - } - pDb->nTransOpen = iLevel; - } - } - - return rc; -} - -int lsm_commit(lsm_db *pDb, int iLevel){ - int rc = LSM_OK; - - assert_db_state( pDb ); - - /* A value less than zero means close the innermost nested transaction. */ - if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); - - if( iLevel<pDb->nTransOpen ){ - if( iLevel==0 ){ - int rc2; - /* Commit the transaction to disk. */ - if( rc==LSM_OK ) rc = lsmLogCommit(pDb); - if( rc==LSM_OK && pDb->eSafety==LSM_SAFETY_FULL ){ - rc = lsmFsSyncLog(pDb->pFS); - } - rc2 = lsmFinishWriteTrans(pDb, (rc==LSM_OK)); - if( rc==LSM_OK ) rc = rc2; - } - pDb->nTransOpen = iLevel; - } - dbReleaseClientSnapshot(pDb); - return rc; -} - -int lsm_rollback(lsm_db *pDb, int iLevel){ - int rc = LSM_OK; - assert_db_state( pDb ); - - if( pDb->nTransOpen ){ - /* A value less than zero means close the innermost nested transaction. */ - if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); - - if( iLevel<=pDb->nTransOpen ){ - TransMark *pMark = &pDb->aTrans[(iLevel==0 ? 0 : iLevel-1)]; - lsmTreeRollback(pDb, &pMark->tree); - if( iLevel ) lsmLogSeek(pDb, &pMark->log); - pDb->nTransOpen = iLevel; - } - - if( pDb->nTransOpen==0 ){ - lsmFinishWriteTrans(pDb, 0); - } - dbReleaseClientSnapshot(pDb); - } - - return rc; -} - -int lsm_get_user_version(lsm_db *pDb, unsigned int *piUsr){ - int rc = LSM_OK; /* Return code */ - - /* Open a read transaction if one is not already open. */ - assert_db_state(pDb); - if( pDb->pShmhdr==0 ){ - assert( pDb->bReadonly ); - rc = lsmBeginRoTrans(pDb); - }else if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Allocate the multi-cursor. */ - if( rc==LSM_OK ){ - *piUsr = pDb->treehdr.iUsrVersion; - } - - dbReleaseClientSnapshot(pDb); - assert_db_state(pDb); - return rc; -} - -int lsm_set_user_version(lsm_db *pDb, unsigned int iUsr){ - int rc = LSM_OK; /* Return code */ - int bCommit = 0; /* True to commit before returning */ - - if( pDb->nTransOpen==0 ){ - bCommit = 1; - rc = lsm_begin(pDb, 1); - } - - if( rc==LSM_OK ){ - pDb->treehdr.iUsrVersion = iUsr; - } - - /* If a transaction was opened at the start of this function, commit it. - ** Or, if an error has occurred, roll it back. */ - if( bCommit ){ - if( rc==LSM_OK ){ - rc = lsm_commit(pDb, 0); - }else{ - lsm_rollback(pDb, 0); - } - } - - return rc; -} diff --git a/ext/lsm1/lsm_mem.c b/ext/lsm1/lsm_mem.c deleted file mode 100644 index 13dd9fe31..000000000 --- a/ext/lsm1/lsm_mem.c +++ /dev/null @@ -1,104 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Helper routines for memory allocation. -*/ -#include "lsmInt.h" - -/* -** The following routines are called internally by LSM sub-routines. In -** this case a valid environment pointer must be supplied. -*/ -void *lsmMalloc(lsm_env *pEnv, size_t N){ - assert( pEnv ); - return pEnv->xMalloc(pEnv, N); -} -void lsmFree(lsm_env *pEnv, void *p){ - assert( pEnv ); - pEnv->xFree(pEnv, p); -} -void *lsmRealloc(lsm_env *pEnv, void *p, size_t N){ - assert( pEnv ); - return pEnv->xRealloc(pEnv, p, N); -} - -/* -** Core memory allocation routines for LSM. -*/ -void *lsm_malloc(lsm_env *pEnv, size_t N){ - return lsmMalloc(pEnv ? pEnv : lsm_default_env(), N); -} -void lsm_free(lsm_env *pEnv, void *p){ - lsmFree(pEnv ? pEnv : lsm_default_env(), p); -} -void *lsm_realloc(lsm_env *pEnv, void *p, size_t N){ - return lsmRealloc(pEnv ? pEnv : lsm_default_env(), p, N); -} - -void *lsmMallocZero(lsm_env *pEnv, size_t N){ - void *pRet; - assert( pEnv ); - pRet = lsmMalloc(pEnv, N); - if( pRet ) memset(pRet, 0, N); - return pRet; -} - -void *lsmMallocRc(lsm_env *pEnv, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc==LSM_OK ){ - pRet = lsmMalloc(pEnv, N); - if( pRet==0 ){ - *pRc = LSM_NOMEM_BKPT; - } - } - return pRet; -} - -void *lsmMallocZeroRc(lsm_env *pEnv, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc==LSM_OK ){ - pRet = lsmMallocZero(pEnv, N); - if( pRet==0 ){ - *pRc = LSM_NOMEM_BKPT; - } - } - return pRet; -} - -void *lsmReallocOrFree(lsm_env *pEnv, void *p, size_t N){ - void *pNew; - pNew = lsm_realloc(pEnv, p, N); - if( !pNew ) lsm_free(pEnv, p); - return pNew; -} - -void *lsmReallocOrFreeRc(lsm_env *pEnv, void *p, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc ){ - lsmFree(pEnv, p); - }else{ - pRet = lsmReallocOrFree(pEnv, p, N); - if( !pRet ) *pRc = LSM_NOMEM_BKPT; - } - return pRet; -} - -char *lsmMallocStrdup(lsm_env *pEnv, const char *zIn){ - int nByte; - char *zRet; - nByte = strlen(zIn); - zRet = lsmMalloc(pEnv, nByte+1); - if( zRet ){ - memcpy(zRet, zIn, nByte+1); - } - return zRet; -} diff --git a/ext/lsm1/lsm_mutex.c b/ext/lsm1/lsm_mutex.c deleted file mode 100644 index cb99b2a61..000000000 --- a/ext/lsm1/lsm_mutex.c +++ /dev/null @@ -1,88 +0,0 @@ -/* -** 2012-01-30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Mutex functions for LSM. -*/ -#include "lsmInt.h" - -/* -** Allocate a new mutex. -*/ -int lsmMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - return pEnv->xMutexNew(pEnv, ppNew); -} - -/* -** Return a handle for one of the static mutexes. -*/ -int lsmMutexStatic(lsm_env *pEnv, int iMutex, lsm_mutex **ppStatic){ - return pEnv->xMutexStatic(pEnv, iMutex, ppStatic); -} - -/* -** Free a mutex allocated by lsmMutexNew(). -*/ -void lsmMutexDel(lsm_env *pEnv, lsm_mutex *pMutex){ - if( pMutex ) pEnv->xMutexDel(pMutex); -} - -/* -** Enter a mutex. -*/ -void lsmMutexEnter(lsm_env *pEnv, lsm_mutex *pMutex){ - pEnv->xMutexEnter(pMutex); -} - -/* -** Attempt to enter a mutex, but do not block. If successful, return zero. -** Otherwise, if the mutex is already held by some other thread and is not -** entered, return non zero. -** -** Each successful call to this function must be matched by a call to -** lsmMutexLeave(). -*/ -int lsmMutexTry(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexTry(pMutex); -} - -/* -** Leave a mutex. -*/ -void lsmMutexLeave(lsm_env *pEnv, lsm_mutex *pMutex){ - pEnv->xMutexLeave(pMutex); -} - -#ifndef NDEBUG -/* -** Return non-zero if the mutex passed as the second argument is held -** by the calling thread, or zero otherwise. If the implementation is not -** able to tell if the mutex is held by the caller, it should return -** non-zero. -** -** This function is only used as part of assert() statements. -*/ -int lsmMutexHeld(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexHeld ? pEnv->xMutexHeld(pMutex) : 1; -} - -/* -** Return non-zero if the mutex passed as the second argument is not -** held by the calling thread, or zero otherwise. If the implementation -** is not able to tell if the mutex is held by the caller, it should -** return non-zero. -** -** This function is only used as part of assert() statements. -*/ -int lsmMutexNotHeld(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexNotHeld ? pEnv->xMutexNotHeld(pMutex) : 1; -} -#endif diff --git a/ext/lsm1/lsm_shared.c b/ext/lsm1/lsm_shared.c deleted file mode 100644 index 09f933848..000000000 --- a/ext/lsm1/lsm_shared.c +++ /dev/null @@ -1,1994 +0,0 @@ -/* -** 2012-01-23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Utilities used to help multiple LSM clients to coexist within the -** same process space. -*/ -#include "lsmInt.h" - -/* -** Global data. All global variables used by code in this file are grouped -** into the following structure instance. -** -** pDatabase: -** Linked list of all Database objects allocated within this process. -** This list may not be traversed without holding the global mutex (see -** functions enterGlobalMutex() and leaveGlobalMutex()). -*/ -static struct SharedData { - Database *pDatabase; /* Linked list of all Database objects */ -} gShared; - -/* -** Database structure. There is one such structure for each distinct -** database accessed by this process. They are stored in the singly linked -** list starting at global variable gShared.pDatabase. Database objects are -** reference counted. Once the number of connections to the associated -** database drops to zero, they are removed from the linked list and deleted. -** -** pFile: -** In multi-process mode, this file descriptor is used to obtain locks -** and to access shared-memory. In single process mode, its only job is -** to hold the exclusive lock on the file. -** -*/ -struct Database { - /* Protected by the global mutex (enterGlobalMutex/leaveGlobalMutex): */ - char *zName; /* Canonical path to database file */ - int nName; /* strlen(zName) */ - int nDbRef; /* Number of associated lsm_db handles */ - Database *pDbNext; /* Next Database structure in global list */ - - /* Protected by the local mutex (pClientMutex) */ - int bReadonly; /* True if Database.pFile is read-only */ - int bMultiProc; /* True if running in multi-process mode */ - lsm_file *pFile; /* Used for locks/shm in multi-proc mode */ - LsmFile *pLsmFile; /* List of deferred closes */ - lsm_mutex *pClientMutex; /* Protects the apShmChunk[] and pConn */ - int nShmChunk; /* Number of entries in apShmChunk[] array */ - void **apShmChunk; /* Array of "shared" memory regions */ - lsm_db *pConn; /* List of connections to this db. */ -}; - -/* -** Functions to enter and leave the global mutex. This mutex is used -** to protect the global linked-list headed at gShared.pDatabase. -*/ -static int enterGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - int rc = lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - if( rc==LSM_OK ) lsmMutexEnter(pEnv, p); - return rc; -} -static void leaveGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - lsmMutexLeave(pEnv, p); -} - -#ifdef LSM_DEBUG -static int holdingGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - return lsmMutexHeld(pEnv, p); -} -#endif - -#if 0 -static void assertNotInFreelist(Freelist *p, int iBlk){ - int i; - for(i=0; i<p->nEntry; i++){ - assert( p->aEntry[i].iBlk!=iBlk ); - } -} -#else -# define assertNotInFreelist(x,y) -#endif - -/* -** Append an entry to the free-list. If (iId==-1), this is a delete. -*/ -int freelistAppend(lsm_db *db, u32 iBlk, i64 iId){ - lsm_env *pEnv = db->pEnv; - Freelist *p; - int i; - - assert( iId==-1 || iId>=0 ); - p = db->bUseFreelist ? db->pFreelist : &db->pWorker->freelist; - - /* Extend the space allocated for the freelist, if required */ - assert( p->nAlloc>=p->nEntry ); - if( p->nAlloc==p->nEntry ){ - int nNew; - int nByte; - FreelistEntry *aNew; - - nNew = (p->nAlloc==0 ? 4 : p->nAlloc*2); - nByte = sizeof(FreelistEntry) * nNew; - aNew = (FreelistEntry *)lsmRealloc(pEnv, p->aEntry, nByte); - if( !aNew ) return LSM_NOMEM_BKPT; - p->nAlloc = nNew; - p->aEntry = aNew; - } - - for(i=0; i<p->nEntry; i++){ - assert( i==0 || p->aEntry[i].iBlk > p->aEntry[i-1].iBlk ); - if( p->aEntry[i].iBlk>=iBlk ) break; - } - - if( i<p->nEntry && p->aEntry[i].iBlk==iBlk ){ - /* Clobber an existing entry */ - p->aEntry[i].iId = iId; - }else{ - /* Insert a new entry into the list */ - int nByte = sizeof(FreelistEntry)*(p->nEntry-i); - memmove(&p->aEntry[i+1], &p->aEntry[i], nByte); - p->aEntry[i].iBlk = iBlk; - p->aEntry[i].iId = iId; - p->nEntry++; - } - - return LSM_OK; -} - -/* -** This function frees all resources held by the Database structure passed -** as the only argument. -*/ -static void freeDatabase(lsm_env *pEnv, Database *p){ - assert( holdingGlobalMutex(pEnv) ); - if( p ){ - /* Free the mutexes */ - lsmMutexDel(pEnv, p->pClientMutex); - - if( p->pFile ){ - lsmEnvClose(pEnv, p->pFile); - } - - /* Free the array of shm pointers */ - lsmFree(pEnv, p->apShmChunk); - - /* Free the memory allocated for the Database struct itself */ - lsmFree(pEnv, p); - } -} - -typedef struct DbTruncateCtx DbTruncateCtx; -struct DbTruncateCtx { - int nBlock; - i64 iInUse; -}; - -static int dbTruncateCb(void *pCtx, int iBlk, i64 iSnapshot){ - DbTruncateCtx *p = (DbTruncateCtx *)pCtx; - if( iBlk!=p->nBlock || (p->iInUse>=0 && iSnapshot>=p->iInUse) ) return 1; - p->nBlock--; - return 0; -} - -static int dbTruncate(lsm_db *pDb, i64 iInUse){ - int rc = LSM_OK; -#if 0 - int i; - DbTruncateCtx ctx; - - assert( pDb->pWorker ); - ctx.nBlock = pDb->pWorker->nBlock; - ctx.iInUse = iInUse; - - rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); - for(i=ctx.nBlock+1; rc==LSM_OK && i<=pDb->pWorker->nBlock; i++){ - rc = freelistAppend(pDb, i, -1); - } - - if( rc==LSM_OK ){ -#ifdef LSM_LOG_FREELIST - if( ctx.nBlock!=pDb->pWorker->nBlock ){ - lsmLogMessage(pDb, 0, - "dbTruncate(): truncated db to %d blocks",ctx.nBlock - ); - } -#endif - pDb->pWorker->nBlock = ctx.nBlock; - } -#endif - return rc; -} - - -/* -** This function is called during database shutdown (when the number of -** connections drops from one to zero). It truncates the database file -** to as small a size as possible without truncating away any blocks that -** contain data. -*/ -static int dbTruncateFile(lsm_db *pDb){ - int rc; - - assert( pDb->pWorker==0 ); - assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) ); - rc = lsmCheckpointLoadWorker(pDb); - - if( rc==LSM_OK ){ - DbTruncateCtx ctx; - - /* Walk the database free-block-list in reverse order. Set ctx.nBlock - ** to the block number of the last block in the database that actually - ** contains data. */ - ctx.nBlock = pDb->pWorker->nBlock; - ctx.iInUse = -1; - rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); - - /* If the last block that contains data is not already the last block in - ** the database file, truncate the database file so that it is. */ - if( rc==LSM_OK ){ - rc = lsmFsTruncateDb( - pDb->pFS, (i64)ctx.nBlock*lsmFsBlockSize(pDb->pFS) - ); - } - } - - lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); - pDb->pWorker = 0; - return rc; -} - -static void doDbDisconnect(lsm_db *pDb){ - int rc; - - if( pDb->bReadonly ){ - lsmShmLock(pDb, LSM_LOCK_DMS3, LSM_LOCK_UNLOCK, 0); - }else{ - /* Block for an exclusive lock on DMS1. This lock serializes all calls - ** to doDbConnect() and doDbDisconnect() across all processes. */ - rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); - if( rc==LSM_OK ){ - - lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); - - /* Try an exclusive lock on DMS2. If successful, this is the last - ** connection to the database. In this case flush the contents of the - ** in-memory tree to disk and write a checkpoint. */ - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 1, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - rc = lsmShmTestLock(pDb, LSM_LOCK_CHECKPOINTER, 1, LSM_LOCK_EXCL); - } - if( rc==LSM_OK ){ - int bReadonly = 0; /* True if there exist read-only conns. */ - - /* Flush the in-memory tree, if required. If there is data to flush, - ** this will create a new client snapshot in Database.pClient. The - ** checkpoint (serialization) of this snapshot may be written to disk - ** by the following block. - ** - ** There is no need to take a WRITER lock here. That there are no - ** other locks on DMS2 guarantees that there are no other read-write - ** connections at this time (and the lock on DMS1 guarantees that - ** no new ones may appear). - */ - rc = lsmTreeLoadHeader(pDb, 0); - if( rc==LSM_OK && (lsmTreeHasOld(pDb) || lsmTreeSize(pDb)>0) ){ - rc = lsmFlushTreeToDisk(pDb); - } - - /* Now check if there are any read-only connections. If there are, - ** then do not truncate the db file or unlink the shared-memory - ** region. */ - if( rc==LSM_OK ){ - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS3, 1, LSM_LOCK_EXCL); - if( rc==LSM_BUSY ){ - bReadonly = 1; - rc = LSM_OK; - } - } - - /* Write a checkpoint to disk. */ - if( rc==LSM_OK ){ - rc = lsmCheckpointWrite(pDb, 0); - } - - /* If the checkpoint was written successfully, delete the log file - ** and, if possible, truncate the database file. */ - if( rc==LSM_OK ){ - int bRotrans = 0; - Database *p = pDb->pDatabase; - - /* The log file may only be deleted if there are no clients - ** read-only clients running rotrans transactions. */ - rc = lsmDetectRoTrans(pDb, &bRotrans); - if( rc==LSM_OK && bRotrans==0 ){ - lsmFsCloseAndDeleteLog(pDb->pFS); - } - - /* The database may only be truncated if there exist no read-only - ** clients - either connected or running rotrans transactions. */ - if( bReadonly==0 && bRotrans==0 ){ - lsmFsUnmap(pDb->pFS); - dbTruncateFile(pDb); - if( p->pFile && p->bMultiProc ){ - lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); - } - } - } - } - } - - if( pDb->iRwclient>=0 ){ - lsmShmLock(pDb, LSM_LOCK_RWCLIENT(pDb->iRwclient), LSM_LOCK_UNLOCK, 0); - pDb->iRwclient = -1; - } - - lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - } - pDb->pShmhdr = 0; -} - -static int doDbConnect(lsm_db *pDb){ - const int nUsMax = 100000; /* Max value for nUs */ - int nUs = 1000; /* us to wait between DMS1 attempts */ - int rc; - - /* Obtain a pointer to the shared-memory header */ - assert( pDb->pShmhdr==0 ); - assert( pDb->bReadonly==0 ); - - /* Block for an exclusive lock on DMS1. This lock serializes all calls - ** to doDbConnect() and doDbDisconnect() across all processes. */ - while( 1 ){ - rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); - if( rc!=LSM_BUSY ) break; - lsmEnvSleep(pDb->pEnv, nUs); - nUs = nUs * 2; - if( nUs>nUsMax ) nUs = nUsMax; - } - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(pDb, 1); - } - if( rc!=LSM_OK ) return rc; - pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; - - /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first - ** and only connection to the database. In this case initialize the - ** shared-memory and run log file recovery. */ - assert( LSM_LOCK_DMS3==1+LSM_LOCK_DMS2 ); - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 2, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - memset(pDb->pShmhdr, 0, sizeof(ShmHeader)); - rc = lsmCheckpointRecover(pDb); - if( rc==LSM_OK ){ - rc = lsmLogRecover(pDb); - } - if( rc==LSM_OK ){ - ShmHeader *pShm = pDb->pShmhdr; - pShm->aReader[0].iLsmId = lsmCheckpointId(pShm->aSnap1, 0); - pShm->aReader[0].iTreeId = pDb->treehdr.iUsedShmid; - } - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - - /* Take a shared lock on DMS2. In multi-process mode this lock "cannot" - ** fail, as connections may only hold an exclusive lock on DMS2 if they - ** first hold an exclusive lock on DMS1. And this connection is currently - ** holding the exclusive lock on DSM1. - ** - ** However, if some other connection has the database open in single-process - ** mode, this operation will fail. In this case, return the error to the - ** caller - the attempt to connect to the db has failed. - */ - if( rc==LSM_OK ){ - rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_SHARED, 0); - } - - /* If anything went wrong, unlock DMS2. Otherwise, try to take an exclusive - ** lock on one of the LSM_LOCK_RWCLIENT() locks. Unlock DMS1 in any case. */ - if( rc!=LSM_OK ){ - pDb->pShmhdr = 0; - }else{ - int i; - for(i=0; i<LSM_LOCK_NRWCLIENT; i++){ - int rc2 = lsmShmLock(pDb, LSM_LOCK_RWCLIENT(i), LSM_LOCK_EXCL, 0); - if( rc2==LSM_OK ) pDb->iRwclient = i; - if( rc2!=LSM_BUSY ){ - rc = rc2; - break; - } - } - } - lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - - return rc; -} - -static int dbOpenSharedFd(lsm_env *pEnv, Database *p, int bRoOk){ - int rc; - - rc = lsmEnvOpen(pEnv, p->zName, 0, &p->pFile); - if( rc==LSM_IOERR && bRoOk ){ - rc = lsmEnvOpen(pEnv, p->zName, LSM_OPEN_READONLY, &p->pFile); - p->bReadonly = 1; - } - - return rc; -} - -/* -** Return a reference to the shared Database handle for the database -** identified by canonical path zName. If this is the first connection to -** the named database, a new Database object is allocated. Otherwise, a -** pointer to an existing object is returned. -** -** If successful, *ppDatabase is set to point to the shared Database -** structure and LSM_OK returned. Otherwise, *ppDatabase is set to NULL -** and and LSM error code returned. -** -** Each successful call to this function should be (eventually) matched -** by a call to lsmDbDatabaseRelease(). -*/ -int lsmDbDatabaseConnect( - lsm_db *pDb, /* Database handle */ - const char *zName /* Full-path to db file */ -){ - lsm_env *pEnv = pDb->pEnv; - int rc; /* Return code */ - Database *p = 0; /* Pointer returned via *ppDatabase */ - int nName = lsmStrlen(zName); - - assert( pDb->pDatabase==0 ); - rc = enterGlobalMutex(pEnv); - if( rc==LSM_OK ){ - - /* Search the global list for an existing object. TODO: Need something - ** better than the memcmp() below to figure out if a given Database - ** object represents the requested file. */ - for(p=gShared.pDatabase; p; p=p->pDbNext){ - if( nName==p->nName && 0==memcmp(zName, p->zName, nName) ) break; - } - - /* If no suitable Database object was found, allocate a new one. */ - if( p==0 ){ - p = (Database *)lsmMallocZeroRc(pEnv, sizeof(Database)+nName+1, &rc); - - /* If the allocation was successful, fill in other fields and - ** allocate the client mutex. */ - if( rc==LSM_OK ){ - p->bMultiProc = pDb->bMultiProc; - p->zName = (char *)&p[1]; - p->nName = nName; - memcpy((void *)p->zName, zName, nName+1); - rc = lsmMutexNew(pEnv, &p->pClientMutex); - } - - /* If nothing has gone wrong so far, open the shared fd. And if that - ** succeeds and this connection requested single-process mode, - ** attempt to take the exclusive lock on DMS2. */ - if( rc==LSM_OK ){ - int bReadonly = (pDb->bReadonly && pDb->bMultiProc); - rc = dbOpenSharedFd(pDb->pEnv, p, bReadonly); - } - - if( rc==LSM_OK && p->bMultiProc==0 ){ - /* Hold an exclusive lock DMS1 while grabbing DMS2. This ensures - ** that any ongoing call to doDbDisconnect() (even one in another - ** process) is finished before proceeding. */ - assert( p->bReadonly==0 ); - rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS1, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS2, LSM_LOCK_EXCL); - lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK); - } - } - - if( rc==LSM_OK ){ - p->pDbNext = gShared.pDatabase; - gShared.pDatabase = p; - }else{ - freeDatabase(pEnv, p); - p = 0; - } - } - - if( p ){ - p->nDbRef++; - } - leaveGlobalMutex(pEnv); - - if( p ){ - lsmMutexEnter(pDb->pEnv, p->pClientMutex); - pDb->pNext = p->pConn; - p->pConn = pDb; - lsmMutexLeave(pDb->pEnv, p->pClientMutex); - } - } - - pDb->pDatabase = p; - if( rc==LSM_OK ){ - assert( p ); - rc = lsmFsOpen(pDb, zName, p->bReadonly); - } - - /* If the db handle is read-write, then connect to the system now. Run - ** recovery as necessary. Or, if this is a read-only database handle, - ** defer attempting to connect to the system until a read-transaction - ** is opened. */ - if( rc==LSM_OK ){ - rc = lsmFsConfigure(pDb); - } - if( rc==LSM_OK && pDb->bReadonly==0 ){ - rc = doDbConnect(pDb); - } - - return rc; -} - -static void dbDeferClose(lsm_db *pDb){ - if( pDb->pFS ){ - LsmFile *pLsmFile; - Database *p = pDb->pDatabase; - pLsmFile = lsmFsDeferClose(pDb->pFS); - pLsmFile->pNext = p->pLsmFile; - p->pLsmFile = pLsmFile; - } -} - -LsmFile *lsmDbRecycleFd(lsm_db *db){ - LsmFile *pRet; - Database *p = db->pDatabase; - lsmMutexEnter(db->pEnv, p->pClientMutex); - if( (pRet = p->pLsmFile)!=0 ){ - p->pLsmFile = pRet->pNext; - } - lsmMutexLeave(db->pEnv, p->pClientMutex); - return pRet; -} - -/* -** Release a reference to a Database object obtained from -** lsmDbDatabaseConnect(). There should be exactly one call to this function -** for each successful call to Find(). -*/ -void lsmDbDatabaseRelease(lsm_db *pDb){ - Database *p = pDb->pDatabase; - if( p ){ - lsm_db **ppDb; - - if( pDb->pShmhdr ){ - doDbDisconnect(pDb); - } - - lsmFsUnmap(pDb->pFS); - lsmMutexEnter(pDb->pEnv, p->pClientMutex); - for(ppDb=&p->pConn; *ppDb!=pDb; ppDb=&((*ppDb)->pNext)); - *ppDb = pDb->pNext; - dbDeferClose(pDb); - lsmMutexLeave(pDb->pEnv, p->pClientMutex); - - enterGlobalMutex(pDb->pEnv); - p->nDbRef--; - if( p->nDbRef==0 ){ - LsmFile *pIter; - LsmFile *pNext; - Database **pp; - - /* Remove the Database structure from the linked list. */ - for(pp=&gShared.pDatabase; *pp!=p; pp=&((*pp)->pDbNext)); - *pp = p->pDbNext; - - /* If they were allocated from the heap, free the shared memory chunks */ - if( p->bMultiProc==0 ){ - int i; - for(i=0; i<p->nShmChunk; i++){ - lsmFree(pDb->pEnv, p->apShmChunk[i]); - } - } - - /* Close any outstanding file descriptors */ - for(pIter=p->pLsmFile; pIter; pIter=pNext){ - pNext = pIter->pNext; - lsmEnvClose(pDb->pEnv, pIter->pFile); - lsmFree(pDb->pEnv, pIter); - } - freeDatabase(pDb->pEnv, p); - } - leaveGlobalMutex(pDb->pEnv); - } -} - -Level *lsmDbSnapshotLevel(Snapshot *pSnapshot){ - return pSnapshot->pLevel; -} - -void lsmDbSnapshotSetLevel(Snapshot *pSnap, Level *pLevel){ - pSnap->pLevel = pLevel; -} - -/* TODO: Shuffle things around to get rid of this */ -static int firstSnapshotInUse(lsm_db *, i64 *); - -/* -** Context object used by the lsmWalkFreelist() utility. -*/ -typedef struct WalkFreelistCtx WalkFreelistCtx; -struct WalkFreelistCtx { - lsm_db *pDb; - int bReverse; - Freelist *pFreelist; - int iFree; - int (*xUsr)(void *, int, i64); /* User callback function */ - void *pUsrctx; /* User callback context */ - int bDone; /* Set to true after xUsr() returns true */ -}; - -/* -** Callback used by lsmWalkFreelist(). -*/ -static int walkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - WalkFreelistCtx *p = (WalkFreelistCtx *)pCtx; - const int iDir = (p->bReverse ? -1 : 1); - Freelist *pFree = p->pFreelist; - - assert( p->bDone==0 ); - assert( iBlk>=0 ); - if( pFree ){ - while( (p->iFree < pFree->nEntry) && p->iFree>=0 ){ - FreelistEntry *pEntry = &pFree->aEntry[p->iFree]; - if( (p->bReverse==0 && pEntry->iBlk>(u32)iBlk) - || (p->bReverse!=0 && pEntry->iBlk<(u32)iBlk) - ){ - break; - }else{ - p->iFree += iDir; - if( pEntry->iId>=0 - && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) - ){ - p->bDone = 1; - return 1; - } - if( pEntry->iBlk==(u32)iBlk ) return 0; - } - } - } - - if( p->xUsr(p->pUsrctx, iBlk, iSnapshot) ){ - p->bDone = 1; - return 1; - } - return 0; -} - -/* -** The database handle passed as the first argument must be the worker -** connection. This function iterates through the contents of the current -** free block list, invoking the supplied callback once for each list -** element. -** -** The difference between this function and lsmSortedWalkFreelist() is -** that lsmSortedWalkFreelist() only considers those free-list elements -** stored within the LSM. This function also merges in any in-memory -** elements. -*/ -int lsmWalkFreelist( - lsm_db *pDb, /* Database handle (must be worker) */ - int bReverse, /* True to iterate from largest to smallest */ - int (*x)(void *, int, i64), /* Callback function */ - void *pCtx /* First argument to pass to callback */ -){ - const int iDir = (bReverse ? -1 : 1); - int rc; - int iCtx; - - WalkFreelistCtx ctx[2]; - - ctx[0].pDb = pDb; - ctx[0].bReverse = bReverse; - ctx[0].pFreelist = &pDb->pWorker->freelist; - if( ctx[0].pFreelist && bReverse ){ - ctx[0].iFree = ctx[0].pFreelist->nEntry-1; - }else{ - ctx[0].iFree = 0; - } - ctx[0].xUsr = walkFreelistCb; - ctx[0].pUsrctx = (void *)&ctx[1]; - ctx[0].bDone = 0; - - ctx[1].pDb = pDb; - ctx[1].bReverse = bReverse; - ctx[1].pFreelist = pDb->pFreelist; - if( ctx[1].pFreelist && bReverse ){ - ctx[1].iFree = ctx[1].pFreelist->nEntry-1; - }else{ - ctx[1].iFree = 0; - } - ctx[1].xUsr = x; - ctx[1].pUsrctx = pCtx; - ctx[1].bDone = 0; - - rc = lsmSortedWalkFreelist(pDb, bReverse, walkFreelistCb, (void *)&ctx[0]); - - if( ctx[0].bDone==0 ){ - for(iCtx=0; iCtx<2; iCtx++){ - int i; - WalkFreelistCtx *p = &ctx[iCtx]; - for(i=p->iFree; - p->pFreelist && rc==LSM_OK && i<p->pFreelist->nEntry && i>=0; - i += iDir - ){ - FreelistEntry *pEntry = &p->pFreelist->aEntry[i]; - if( pEntry->iId>=0 && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) ){ - return LSM_OK; - } - } - } - } - - return rc; -} - - -typedef struct FindFreeblockCtx FindFreeblockCtx; -struct FindFreeblockCtx { - i64 iInUse; - int iRet; - int bNotOne; -}; - -static int findFreeblockCb(void *pCtx, int iBlk, i64 iSnapshot){ - FindFreeblockCtx *p = (FindFreeblockCtx *)pCtx; - if( iSnapshot<p->iInUse && (iBlk!=1 || p->bNotOne==0) ){ - p->iRet = iBlk; - return 1; - } - return 0; -} - -static int findFreeblock(lsm_db *pDb, i64 iInUse, int bNotOne, int *piRet){ - int rc; /* Return code */ - FindFreeblockCtx ctx; /* Context object */ - - ctx.iInUse = iInUse; - ctx.iRet = 0; - ctx.bNotOne = bNotOne; - rc = lsmWalkFreelist(pDb, 0, findFreeblockCb, (void *)&ctx); - *piRet = ctx.iRet; - - return rc; -} - -/* -** Allocate a new database file block to write data to, either by extending -** the database file or by recycling a free-list entry. The worker snapshot -** must be held in order to call this function. -** -** If successful, *piBlk is set to the block number allocated and LSM_OK is -** returned. Otherwise, *piBlk is zeroed and an lsm error code returned. -*/ -int lsmBlockAllocate(lsm_db *pDb, int iBefore, int *piBlk){ - Snapshot *p = pDb->pWorker; - int iRet = 0; /* Block number of allocated block */ - int rc = LSM_OK; - i64 iInUse = 0; /* Snapshot id still in use */ - i64 iSynced = 0; /* Snapshot id synced to disk */ - - assert( p ); - -#ifdef LSM_LOG_FREELIST - { - static int nCall = 0; - char *zFree = 0; - nCall++; - rc = lsmInfoFreelist(pDb, &zFree); - if( rc!=LSM_OK ) return rc; - lsmLogMessage(pDb, 0, "lsmBlockAllocate(): %d freelist: %s", nCall, zFree); - lsmFree(pDb->pEnv, zFree); - } -#endif - - /* Set iInUse to the smallest snapshot id that is either: - ** - ** * Currently in use by a database client, - ** * May be used by a database client in the future, or - ** * Is the most recently checkpointed snapshot (i.e. the one that will - ** be used following recovery if a failure occurs at this point). - */ - rc = lsmCheckpointSynced(pDb, &iSynced, 0, 0); - if( rc==LSM_OK && iSynced==0 ) iSynced = p->iId; - iInUse = iSynced; - if( rc==LSM_OK && pDb->iReader>=0 ){ - assert( pDb->pClient ); - iInUse = LSM_MIN(iInUse, pDb->pClient->iId); - } - if( rc==LSM_OK ) rc = firstSnapshotInUse(pDb, &iInUse); - -#ifdef LSM_LOG_FREELIST - { - lsmLogMessage(pDb, 0, "lsmBlockAllocate(): " - "snapshot-in-use: %lld (iSynced=%lld) (client-id=%lld)", - iInUse, iSynced, (pDb->iReader>=0 ? pDb->pClient->iId : 0) - ); - } -#endif - - - /* Unless there exists a read-only transaction (which prevents us from - ** recycling any blocks regardless, query the free block list for a - ** suitable block to reuse. - ** - ** It might seem more natural to check for a read-only transaction at - ** the start of this function. However, it is better do wait until after - ** the call to lsmCheckpointSynced() to do so. - */ - if( rc==LSM_OK ){ - int bRotrans; - rc = lsmDetectRoTrans(pDb, &bRotrans); - - if( rc==LSM_OK && bRotrans==0 ){ - rc = findFreeblock(pDb, iInUse, (iBefore>0), &iRet); - } - } - - if( iBefore>0 && (iRet<=0 || iRet>=iBefore) ){ - iRet = 0; - - }else if( rc==LSM_OK ){ - /* If a block was found in the free block list, use it and remove it from - ** the list. Otherwise, if no suitable block was found, allocate one from - ** the end of the file. */ - if( iRet>0 ){ -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, 0, - "reusing block %d (snapshot-in-use=%lld)", iRet, iInUse); -#endif - rc = freelistAppend(pDb, iRet, -1); - if( rc==LSM_OK ){ - rc = dbTruncate(pDb, iInUse); - } - }else{ - iRet = ++(p->nBlock); -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, 0, "extending file to %d blocks", iRet); -#endif - } - } - - assert( iBefore>0 || iRet>0 || rc!=LSM_OK ); - *piBlk = iRet; - return rc; -} - -/* -** Free a database block. The worker snapshot must be held in order to call -** this function. -** -** If successful, LSM_OK is returned. Otherwise, an lsm error code (e.g. -** LSM_NOMEM). -*/ -int lsmBlockFree(lsm_db *pDb, int iBlk){ - Snapshot *p = pDb->pWorker; - assert( lsmShmAssertWorker(pDb) ); - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, LSM_OK, "lsmBlockFree(): Free block %d", iBlk); -#endif - - return freelistAppend(pDb, iBlk, p->iId); -} - -/* -** Refree a database block. The worker snapshot must be held in order to call -** this function. -** -** Refreeing is required when a block is allocated using lsmBlockAllocate() -** but then not used. This function is used to push the block back onto -** the freelist. Refreeing a block is different from freeing is, as a refreed -** block may be reused immediately. Whereas a freed block can not be reused -** until (at least) after the next checkpoint. -*/ -int lsmBlockRefree(lsm_db *pDb, int iBlk){ - int rc = LSM_OK; /* Return code */ - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, LSM_OK, "lsmBlockRefree(): Refree block %d", iBlk); -#endif - - rc = freelistAppend(pDb, iBlk, 0); - return rc; -} - -/* -** If required, copy a database checkpoint from shared memory into the -** database itself. -** -** The WORKER lock must not be held when this is called. This is because -** this function may indirectly call fsync(). And the WORKER lock should -** not be held that long (in case it is required by a client flushing an -** in-memory tree to disk). -*/ -int lsmCheckpointWrite(lsm_db *pDb, u32 *pnWrite){ - int rc; /* Return Code */ - u32 nWrite = 0; - - assert( pDb->pWorker==0 ); - assert( 1 || pDb->pClient==0 ); - assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK) ); - - rc = lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_EXCL, 0); - if( rc!=LSM_OK ) return rc; - - rc = lsmCheckpointLoad(pDb, 0); - if( rc==LSM_OK ){ - int nBlock = lsmCheckpointNBlock(pDb->aSnapshot); - ShmHeader *pShm = pDb->pShmhdr; - int bDone = 0; /* True if checkpoint is already stored */ - - /* Check if this checkpoint has already been written to the database - ** file. If so, set variable bDone to true. */ - if( pShm->iMetaPage ){ - MetaPage *pPg; /* Meta page */ - u8 *aData; /* Meta-page data buffer */ - int nData; /* Size of aData[] in bytes */ - i64 iCkpt; /* Id of checkpoint just loaded */ - i64 iDisk = 0; /* Id of checkpoint already stored in db */ - iCkpt = lsmCheckpointId(pDb->aSnapshot, 0); - rc = lsmFsMetaPageGet(pDb->pFS, 0, pShm->iMetaPage, &pPg); - if( rc==LSM_OK ){ - aData = lsmFsMetaPageData(pPg, &nData); - iDisk = lsmCheckpointId((u32 *)aData, 1); - nWrite = lsmCheckpointNWrite((u32 *)aData, 1); - lsmFsMetaPageRelease(pPg); - } - bDone = (iDisk>=iCkpt); - } - - if( rc==LSM_OK && bDone==0 ){ - int iMeta = (pShm->iMetaPage % 2) + 1; - if( pDb->eSafety!=LSM_SAFETY_OFF ){ - rc = lsmFsSyncDb(pDb->pFS, nBlock); - } - if( rc==LSM_OK ) rc = lsmCheckpointStore(pDb, iMeta); - if( rc==LSM_OK && pDb->eSafety!=LSM_SAFETY_OFF){ - rc = lsmFsSyncDb(pDb->pFS, 0); - } - if( rc==LSM_OK ){ - pShm->iMetaPage = iMeta; - nWrite = lsmCheckpointNWrite(pDb->aSnapshot, 0) - nWrite; - } -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, 0, "finish checkpoint %d", - (int)lsmCheckpointId(pDb->aSnapshot, 0) - ); -#endif - } - } - - lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_UNLOCK, 0); - if( pnWrite && rc==LSM_OK ) *pnWrite = nWrite; - return rc; -} - -int lsmBeginWork(lsm_db *pDb){ - int rc; - - /* Attempt to take the WORKER lock */ - rc = lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL, 0); - - /* Deserialize the current worker snapshot */ - if( rc==LSM_OK ){ - rc = lsmCheckpointLoadWorker(pDb); - } - return rc; -} - -void lsmFreeSnapshot(lsm_env *pEnv, Snapshot *p){ - if( p ){ - lsmSortedFreeLevel(pEnv, p->pLevel); - lsmFree(pEnv, p->freelist.aEntry); - lsmFree(pEnv, p->redirect.a); - lsmFree(pEnv, p); - } -} - -/* -** Attempt to populate one of the read-lock slots to contain lock values -** iLsm/iShm. Or, if such a slot exists already, this function is a no-op. -** -** It is not an error if no slot can be populated because the write-lock -** cannot be obtained. If any other error occurs, return an LSM error code. -** Otherwise, LSM_OK. -** -** This function is called at various points to try to ensure that there -** always exists at least one read-lock slot that can be used by a read-only -** client. And so that, in the usual case, there is an "exact match" available -** whenever a read transaction is opened by any client. At present this -** function is called when: -** -** * A write transaction that called lsmTreeDiscardOld() is committed, and -** * Whenever the working snapshot is updated (i.e. lsmFinishWork()). -*/ -static int dbSetReadLock(lsm_db *db, i64 iLsm, u32 iShm){ - int rc = LSM_OK; - ShmHeader *pShm = db->pShmhdr; - int i; - - /* Check if there is already a slot containing the required values. */ - for(i=0; i<LSM_LOCK_NREADER; i++){ - ShmReader *p = &pShm->aReader[i]; - if( p->iLsmId==iLsm && p->iTreeId==iShm ) return LSM_OK; - } - - /* Iterate through all read-lock slots, attempting to take a write-lock - ** on each of them. If a write-lock succeeds, populate the locked slot - ** with the required values and break out of the loop. */ - for(i=0; rc==LSM_OK && i<LSM_LOCK_NREADER; i++){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_BUSY ){ - rc = LSM_OK; - }else{ - ShmReader *p = &pShm->aReader[i]; - p->iLsmId = iLsm; - p->iTreeId = iShm; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - break; - } - } - - return rc; -} - -/* -** Release the read-lock currently held by connection db. -*/ -int dbReleaseReadlock(lsm_db *db){ - int rc = LSM_OK; - if( db->iReader>=0 ){ - rc = lsmShmLock(db, LSM_LOCK_READER(db->iReader), LSM_LOCK_UNLOCK, 0); - db->iReader = -1; - } - db->bRoTrans = 0; - return rc; -} - - -/* -** Argument bFlush is true if the contents of the in-memory tree has just -** been flushed to disk. The significance of this is that once the snapshot -** created to hold the updated state of the database is synced to disk, log -** file space can be recycled. -*/ -void lsmFinishWork(lsm_db *pDb, int bFlush, int *pRc){ - int rc = *pRc; - assert( rc!=0 || pDb->pWorker ); - if( pDb->pWorker ){ - /* If no error has occurred, serialize the worker snapshot and write - ** it to shared memory. */ - if( rc==LSM_OK ){ - rc = lsmSaveWorker(pDb, bFlush); - } - - /* Assuming no error has occurred, update a read lock slot with the - ** new snapshot id (see comments above function dbSetReadLock()). */ - if( rc==LSM_OK ){ - if( pDb->iReader<0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } - if( rc==LSM_OK ){ - rc = dbSetReadLock(pDb, pDb->pWorker->iId, pDb->treehdr.iUsedShmid); - } - } - - /* Free the snapshot object. */ - lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); - pDb->pWorker = 0; - } - - lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK, 0); - *pRc = rc; -} - -/* -** Called when recovery is finished. -*/ -int lsmFinishRecovery(lsm_db *pDb){ - lsmTreeEndTransaction(pDb, 1); - return LSM_OK; -} - -/* -** Check if the currently configured compression functions -** (LSM_CONFIG_SET_COMPRESSION) are compatible with a database that has its -** compression id set to iReq. Compression routines are compatible if iReq -** is zero (indicating the database is empty), or if it is equal to the -** compression id of the configured compression routines. -** -** If the check shows that the current compression are incompatible and there -** is a compression factory registered, give it a chance to install new -** compression routines. -** -** If, after any registered factory is invoked, the compression functions -** are still incompatible, return LSM_MISMATCH. Otherwise, LSM_OK. -*/ -int lsmCheckCompressionId(lsm_db *pDb, u32 iReq){ - if( iReq!=LSM_COMPRESSION_EMPTY && pDb->compress.iId!=iReq ){ - if( pDb->factory.xFactory ){ - pDb->bInFactory = 1; - pDb->factory.xFactory(pDb->factory.pCtx, pDb, iReq); - pDb->bInFactory = 0; - } - if( pDb->compress.iId!=iReq ){ - /* Incompatible */ - return LSM_MISMATCH; - } - } - /* Compatible */ - return LSM_OK; -} - -/* -** Begin a read transaction. This function is a no-op if the connection -** passed as the only argument already has an open read transaction. -*/ -int lsmBeginReadTrans(lsm_db *pDb){ - const int MAX_READLOCK_ATTEMPTS = 10; - const int nMaxAttempt = (pDb->bRoTrans ? 1 : MAX_READLOCK_ATTEMPTS); - - int rc = LSM_OK; /* Return code */ - int iAttempt = 0; - - assert( pDb->pWorker==0 ); - - while( rc==LSM_OK && pDb->iReader<0 && (iAttempt++)<nMaxAttempt ){ - int iTreehdr = 0; - int iSnap = 0; - assert( pDb->pCsr==0 && pDb->nTransOpen==0 ); - - /* Load the in-memory tree header. */ - rc = lsmTreeLoadHeader(pDb, &iTreehdr); - - /* Load the database snapshot */ - if( rc==LSM_OK ){ - if( lsmCheckpointClientCacheOk(pDb)==0 ){ - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - lsmMCursorFreeCache(pDb); - lsmFsPurgeCache(pDb->pFS); - rc = lsmCheckpointLoad(pDb, &iSnap); - }else{ - iSnap = 1; - } - } - - /* Take a read-lock on the tree and snapshot just loaded. Then check - ** that the shared-memory still contains the same values. If so, proceed. - ** Otherwise, relinquish the read-lock and retry the whole procedure - ** (starting with loading the in-memory tree header). */ - if( rc==LSM_OK ){ - u32 iShmMax = pDb->treehdr.iUsedShmid; - u32 iShmMin = pDb->treehdr.iNextShmid+1-LSM_MAX_SHMCHUNKS; - rc = lsmReadlock( - pDb, lsmCheckpointId(pDb->aSnapshot, 0), iShmMin, iShmMax - ); - if( rc==LSM_OK ){ - if( lsmTreeLoadHeaderOk(pDb, iTreehdr) - && lsmCheckpointLoadOk(pDb, iSnap) - ){ - /* Read lock has been successfully obtained. Deserialize the - ** checkpoint just loaded. TODO: This will be removed after - ** lsm_sorted.c is changed to work directly from the serialized - ** version of the snapshot. */ - if( pDb->pClient==0 ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); - } - assert( (rc==LSM_OK)==(pDb->pClient!=0) ); - assert( pDb->iReader>=0 ); - - /* Check that the client has the right compression hooks loaded. - ** If not, set rc to LSM_MISMATCH. */ - if( rc==LSM_OK ){ - rc = lsmCheckCompressionId(pDb, pDb->pClient->iCmpId); - } - }else{ - rc = dbReleaseReadlock(pDb); - } - } - - if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } -#if 0 -if( rc==LSM_OK && pDb->pClient ){ - fprintf(stderr, - "reading %p: snapshot:%d used-shmid:%d trans-id:%d iOldShmid=%d\n", - (void *)pDb, - (int)pDb->pClient->iId, (int)pDb->treehdr.iUsedShmid, - (int)pDb->treehdr.root.iTransId, - (int)pDb->treehdr.iOldShmid - ); -} -#endif - } - - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); - } - if( rc!=LSM_OK ){ - dbReleaseReadlock(pDb); - } - if( pDb->pClient==0 && rc==LSM_OK ) rc = LSM_BUSY; - return rc; -} - -/* -** This function is used by a read-write connection to determine if there -** are currently one or more read-only transactions open on the database -** (in this context a read-only transaction is one opened by a read-only -** connection on a non-live database). -** -** If no error occurs, LSM_OK is returned and *pbExists is set to true if -** some other connection has a read-only transaction open, or false -** otherwise. If an error occurs an LSM error code is returned and the final -** value of *pbExist is undefined. -*/ -int lsmDetectRoTrans(lsm_db *db, int *pbExist){ - int rc; - - /* Only a read-write connection may use this function. */ - assert( db->bReadonly==0 ); - - rc = lsmShmTestLock(db, LSM_LOCK_ROTRANS, 1, LSM_LOCK_EXCL); - if( rc==LSM_BUSY ){ - *pbExist = 1; - rc = LSM_OK; - }else{ - *pbExist = 0; - } - - return rc; -} - -/* -** db is a read-only database handle in the disconnected state. This function -** attempts to open a read-transaction on the database. This may involve -** connecting to the database system (opening shared memory etc.). -*/ -int lsmBeginRoTrans(lsm_db *db){ - int rc = LSM_OK; - - assert( db->bReadonly && db->pShmhdr==0 ); - assert( db->iReader<0 ); - - if( db->bRoTrans==0 ){ - - /* Attempt a shared-lock on DMS1. */ - rc = lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_SHARED, 0); - if( rc!=LSM_OK ) return rc; - - rc = lsmShmTestLock( - db, LSM_LOCK_RWCLIENT(0), LSM_LOCK_NREADER, LSM_LOCK_SHARED - ); - if( rc==LSM_OK ){ - /* System is not live. Take a SHARED lock on the ROTRANS byte and - ** release DMS1. Locking ROTRANS tells all read-write clients that they - ** may not recycle any disk space from within the database or log files, - ** as a read-only client may be using it. */ - rc = lsmShmLock(db, LSM_LOCK_ROTRANS, LSM_LOCK_SHARED, 0); - lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - - if( rc==LSM_OK ){ - db->bRoTrans = 1; - rc = lsmShmCacheChunks(db, 1); - if( rc==LSM_OK ){ - db->pShmhdr = (ShmHeader *)db->apShm[0]; - memset(db->pShmhdr, 0, sizeof(ShmHeader)); - rc = lsmCheckpointRecover(db); - if( rc==LSM_OK ){ - rc = lsmLogRecover(db); - } - } - } - }else if( rc==LSM_BUSY ){ - /* System is live! */ - rc = lsmShmLock(db, LSM_LOCK_DMS3, LSM_LOCK_SHARED, 0); - lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(db, 1); - if( rc==LSM_OK ){ - db->pShmhdr = (ShmHeader *)db->apShm[0]; - } - } - } - - /* In 'lsm_open()' we don't update the page and block sizes in the - ** Filesystem for 'readonly' connection. Because member 'db->pShmhdr' is a - ** nullpointer, this prevents loading a checkpoint. Now that the system is - ** live this member should be set. So we can update both values in - ** the Filesystem. - ** - ** Configure the file-system connection with the page-size and block-size - ** of this database. Even if the database file is zero bytes in size - ** on disk, these values have been set in shared-memory by now, and so - ** are guaranteed not to change during the lifetime of this connection. */ - if( LSM_OK==rc - && 0==lsmCheckpointClientCacheOk(db) - && LSM_OK==(rc=lsmCheckpointLoad(db, 0)) - ){ - lsmFsSetPageSize(db->pFS, lsmCheckpointPgsz(db->aSnapshot)); - lsmFsSetBlockSize(db->pFS, lsmCheckpointBlksz(db->aSnapshot)); - } - - if( rc==LSM_OK ){ - rc = lsmBeginReadTrans(db); - } - } - - return rc; -} - -/* -** Close the currently open read transaction. -*/ -void lsmFinishReadTrans(lsm_db *pDb){ - - /* Worker connections should not be closing read transactions. And - ** read transactions should only be closed after all cursors and write - ** transactions have been closed. Finally pClient should be non-NULL - ** only iff pDb->iReader>=0. */ - assert( pDb->pWorker==0 ); - assert( pDb->pCsr==0 && pDb->nTransOpen==0 ); - - if( pDb->bRoTrans ){ - int i; - for(i=0; i<pDb->nShm; i++){ - lsmFree(pDb->pEnv, pDb->apShm[i]); - } - lsmFree(pDb->pEnv, pDb->apShm); - pDb->apShm = 0; - pDb->nShm = 0; - pDb->pShmhdr = 0; - - lsmShmLock(pDb, LSM_LOCK_ROTRANS, LSM_LOCK_UNLOCK, 0); - } - dbReleaseReadlock(pDb); -} - -/* -** Open a write transaction. -*/ -int lsmBeginWriteTrans(lsm_db *pDb){ - int rc = LSM_OK; /* Return code */ - ShmHeader *pShm = pDb->pShmhdr; /* Shared memory header */ - - assert( pDb->nTransOpen==0 ); - assert( pDb->bDiscardOld==0 ); - assert( pDb->bReadonly==0 ); - - /* If there is no read-transaction open, open one now. */ - if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Attempt to take the WRITER lock */ - if( rc==LSM_OK ){ - rc = lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL, 0); - } - - /* If the previous writer failed mid-transaction, run emergency rollback. */ - if( rc==LSM_OK && pShm->bWriter ){ - rc = lsmTreeRepair(pDb); - if( rc==LSM_OK ) pShm->bWriter = 0; - } - - /* Check that this connection is currently reading from the most recent - ** version of the database. If not, return LSM_BUSY. */ - if( rc==LSM_OK && memcmp(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)) ){ - rc = LSM_BUSY; - } - - if( rc==LSM_OK ){ - rc = lsmLogBegin(pDb); - } - - /* If everything was successful, set the "transaction-in-progress" flag - ** and return LSM_OK. Otherwise, if some error occurred, relinquish the - ** WRITER lock and return an error code. */ - if( rc==LSM_OK ){ - TreeHeader *p = &pDb->treehdr; - pShm->bWriter = 1; - p->root.iTransId++; - if( lsmTreeHasOld(pDb) && p->iOldLog==pDb->pClient->iLogOff ){ - lsmTreeDiscardOld(pDb); - pDb->bDiscardOld = 1; - } - }else{ - lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); - if( pDb->pCsr==0 ) lsmFinishReadTrans(pDb); - } - return rc; -} - -/* -** End the current write transaction. The connection is left with an open -** read transaction. It is an error to call this if there is no open write -** transaction. -** -** If the transaction was committed, then a commit record has already been -** written into the log file when this function is called. Or, if the -** transaction was rolled back, both the log file and in-memory tree -** structure have already been restored. In either case, this function -** merely releases locks and other resources held by the write-transaction. -** -** LSM_OK is returned if successful, or an LSM error code otherwise. -*/ -int lsmFinishWriteTrans(lsm_db *pDb, int bCommit){ - int rc = LSM_OK; - int bFlush = 0; - - lsmLogEnd(pDb, bCommit); - if( rc==LSM_OK && bCommit && lsmTreeSize(pDb)>pDb->nTreeLimit ){ - bFlush = 1; - lsmTreeMakeOld(pDb); - } - lsmTreeEndTransaction(pDb, bCommit); - - if( rc==LSM_OK ){ - if( bFlush && pDb->bAutowork ){ - rc = lsmSortedAutoWork(pDb, 1); - }else if( bCommit && pDb->bDiscardOld ){ - rc = dbSetReadLock(pDb, pDb->pClient->iId, pDb->treehdr.iUsedShmid); - } - } - pDb->bDiscardOld = 0; - lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); - - if( bFlush && pDb->bAutowork==0 && pDb->xWork ){ - pDb->xWork(pDb, pDb->pWorkCtx); - } - return rc; -} - - -/* -** Return non-zero if the caller is holding the client mutex. -*/ -#ifdef LSM_DEBUG -int lsmHoldingClientMutex(lsm_db *pDb){ - return lsmMutexHeld(pDb->pEnv, pDb->pDatabase->pClientMutex); -} -#endif - -static int slotIsUsable(ShmReader *p, i64 iLsm, u32 iShmMin, u32 iShmMax){ - return( - p->iLsmId && p->iLsmId<=iLsm - && shm_sequence_ge(iShmMax, p->iTreeId) - && shm_sequence_ge(p->iTreeId, iShmMin) - ); -} - -/* -** Obtain a read-lock on database version identified by the combination -** of snapshot iLsm and tree iTree. Return LSM_OK if successful, or -** an LSM error code otherwise. -*/ -int lsmReadlock(lsm_db *db, i64 iLsm, u32 iShmMin, u32 iShmMax){ - int rc = LSM_OK; - ShmHeader *pShm = db->pShmhdr; - int i; - - assert( db->iReader<0 ); - assert( shm_sequence_ge(iShmMax, iShmMin) ); - - /* This is a no-op if the read-only transaction flag is set. */ - if( db->bRoTrans ){ - db->iReader = 0; - return LSM_OK; - } - - /* Search for an exact match. */ - for(i=0; db->iReader<0 && rc==LSM_OK && i<LSM_LOCK_NREADER; i++){ - ShmReader *p = &pShm->aReader[i]; - if( p->iLsmId==iLsm && p->iTreeId==iShmMax ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - if( rc==LSM_OK && p->iLsmId==iLsm && p->iTreeId==iShmMax ){ - db->iReader = i; - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } - } - - /* Try to obtain a write-lock on each slot, in order. If successful, set - ** the slot values to iLsm/iTree. */ - for(i=0; db->iReader<0 && rc==LSM_OK && i<LSM_LOCK_NREADER; i++){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_BUSY ){ - rc = LSM_OK; - }else{ - ShmReader *p = &pShm->aReader[i]; - p->iLsmId = iLsm; - p->iTreeId = iShmMax; - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - assert( rc!=LSM_BUSY ); - if( rc==LSM_OK ) db->iReader = i; - } - } - - /* Search for any usable slot */ - for(i=0; db->iReader<0 && rc==LSM_OK && i<LSM_LOCK_NREADER; i++){ - ShmReader *p = &pShm->aReader[i]; - if( slotIsUsable(p, iLsm, iShmMin, iShmMax) ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - if( rc==LSM_OK && slotIsUsable(p, iLsm, iShmMin, iShmMax) ){ - db->iReader = i; - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } - } - - if( rc==LSM_OK && db->iReader<0 ){ - rc = LSM_BUSY; - } - return rc; -} - -/* -** This is used to check if there exists a read-lock locking a particular -** version of either the in-memory tree or database file. -** -** If iLsmId is non-zero, then it is a snapshot id. If there exists a -** read-lock using this snapshot or newer, set *pbInUse to true. Or, -** if there is no such read-lock, set it to false. -** -** Or, if iLsmId is zero, then iShmid is a shared-memory sequence id. -** Search for a read-lock using this sequence id or newer. etc. -*/ -static int isInUse(lsm_db *db, i64 iLsmId, u32 iShmid, int *pbInUse){ - ShmHeader *pShm = db->pShmhdr; - int i; - int rc = LSM_OK; - - for(i=0; rc==LSM_OK && i<LSM_LOCK_NREADER; i++){ - ShmReader *p = &pShm->aReader[i]; - if( p->iLsmId ){ - if( (iLsmId!=0 && p->iLsmId!=0 && iLsmId>=p->iLsmId) - || (iLsmId==0 && shm_sequence_ge(p->iTreeId, iShmid)) - ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_OK ){ - p->iLsmId = 0; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - } - } - } - } - - if( rc==LSM_BUSY ){ - *pbInUse = 1; - return LSM_OK; - } - *pbInUse = 0; - return rc; -} - -/* -** This function is called by worker connections to determine the smallest -** snapshot id that is currently in use by a database client. The worker -** connection uses this result to determine whether or not it is safe to -** recycle a database block. -*/ -static int firstSnapshotInUse( - lsm_db *db, /* Database handle */ - i64 *piInUse /* IN/OUT: Smallest snapshot id in use */ -){ - ShmHeader *pShm = db->pShmhdr; - i64 iInUse = *piInUse; - int i; - - assert( iInUse>0 ); - for(i=0; i<LSM_LOCK_NREADER; i++){ - ShmReader *p = &pShm->aReader[i]; - if( p->iLsmId ){ - i64 iThis = p->iLsmId; - if( iThis!=0 && iInUse>iThis ){ - int rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_OK ){ - p->iLsmId = 0; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - }else if( rc==LSM_BUSY ){ - iInUse = iThis; - }else{ - /* Some error other than LSM_BUSY. Return the error code to - ** the caller in this case. */ - return rc; - } - } - } - } - - *piInUse = iInUse; - return LSM_OK; -} - -int lsmTreeInUse(lsm_db *db, u32 iShmid, int *pbInUse){ - if( db->treehdr.iUsedShmid==iShmid ){ - *pbInUse = 1; - return LSM_OK; - } - return isInUse(db, 0, iShmid, pbInUse); -} - -int lsmLsmInUse(lsm_db *db, i64 iLsmId, int *pbInUse){ - if( db->pClient && db->pClient->iId<=iLsmId ){ - *pbInUse = 1; - return LSM_OK; - } - return isInUse(db, iLsmId, 0, pbInUse); -} - -/* -** This function may only be called after a successful call to -** lsmDbDatabaseConnect(). It returns true if the connection is in -** multi-process mode, or false otherwise. -*/ -int lsmDbMultiProc(lsm_db *pDb){ - return pDb->pDatabase && pDb->pDatabase->bMultiProc; -} - - -/************************************************************************* -************************************************************************** -************************************************************************** -************************************************************************** -************************************************************************** -*************************************************************************/ - -/* -** Ensure that database connection db has cached pointers to at least the -** first nChunk chunks of shared memory. -*/ -int lsmShmCacheChunks(lsm_db *db, int nChunk){ - int rc = LSM_OK; - if( nChunk>db->nShm ){ - static const int NINCR = 16; - Database *p = db->pDatabase; - lsm_env *pEnv = db->pEnv; - int nAlloc; - int i; - - /* Ensure that the db->apShm[] array is large enough. If an attempt to - ** allocate memory fails, return LSM_NOMEM immediately. The apShm[] array - ** is always extended in multiples of 16 entries - so the actual allocated - ** size can be inferred from nShm. */ - nAlloc = ((db->nShm + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*nAlloc); - if( !apShm ) return LSM_NOMEM_BKPT; - db->apShm = apShm; - } - - if( db->bRoTrans ){ - for(i=db->nShm; rc==LSM_OK && i<nChunk; i++){ - db->apShm[i] = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - db->nShm++; - } - - }else{ - - /* Enter the client mutex */ - lsmMutexEnter(pEnv, p->pClientMutex); - - /* Extend the Database objects apShmChunk[] array if necessary. Using the - ** same pattern as for the lsm_db.apShm[] array above. */ - nAlloc = ((p->nShmChunk + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, p->apShmChunk, sizeof(void*)*nAlloc); - if( !apShm ){ - rc = LSM_NOMEM_BKPT; - break; - } - p->apShmChunk = apShm; - } - - for(i=db->nShm; rc==LSM_OK && i<nChunk; i++){ - if( i>=p->nShmChunk ){ - void *pChunk = 0; - if( p->bMultiProc==0 ){ - /* Single process mode */ - pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - }else{ - /* Multi-process mode */ - rc = lsmEnvShmMap(pEnv, p->pFile, i, LSM_SHM_CHUNK_SIZE, &pChunk); - } - if( rc==LSM_OK ){ - p->apShmChunk[i] = pChunk; - p->nShmChunk++; - } - } - if( rc==LSM_OK ){ - db->apShm[i] = p->apShmChunk[i]; - db->nShm++; - } - } - - /* Release the client mutex */ - lsmMutexLeave(pEnv, p->pClientMutex); - } - } - - return rc; -} - -static int lockSharedFile(lsm_env *pEnv, Database *p, int iLock, int eOp){ - int rc = LSM_OK; - if( p->bMultiProc ){ - rc = lsmEnvLock(pEnv, p->pFile, iLock, eOp); - } - return rc; -} - -/* -** Test if it would be possible for connection db to obtain a lock of type -** eType on the nLock locks starting at iLock. If so, return LSM_OK. If it -** would not be possible to obtain the lock due to a lock held by another -** connection, return LSM_BUSY. If an IO or other error occurs (i.e. in the -** lsm_env.xTestLock function), return some other LSM error code. -** -** Note that this function never actually locks the database - it merely -** queries the system to see if there exists a lock that would prevent -** it from doing so. -*/ -int lsmShmTestLock( - lsm_db *db, - int iLock, - int nLock, - int eOp -){ - int rc = LSM_OK; - lsm_db *pIter; - Database *p = db->pDatabase; - int i; - u64 mask = 0; - - for(i=iLock; i<(iLock+nLock); i++){ - mask |= ((u64)1 << (iLock-1)); - if( eOp==LSM_LOCK_EXCL ) mask |= ((u64)1 << (iLock+32-1)); - } - - lsmMutexEnter(db->pEnv, p->pClientMutex); - for(pIter=p->pConn; pIter; pIter=pIter->pNext){ - if( pIter!=db && (pIter->mLock & mask) ){ - assert( pIter!=db ); - break; - } - } - - if( pIter ){ - rc = LSM_BUSY; - }else if( p->bMultiProc ){ - rc = lsmEnvTestLock(db->pEnv, p->pFile, iLock, nLock, eOp); - } - - lsmMutexLeave(db->pEnv, p->pClientMutex); - return rc; -} - -/* -** Attempt to obtain the lock identified by the iLock and bExcl parameters. -** If successful, return LSM_OK. If the lock cannot be obtained because -** there exists some other conflicting lock, return LSM_BUSY. If some other -** error occurs, return an LSM error code. -** -** Parameter iLock must be one of LSM_LOCK_WRITER, WORKER or CHECKPOINTER, -** or else a value returned by the LSM_LOCK_READER macro. -*/ -int lsmShmLock( - lsm_db *db, - int iLock, - int eOp, /* One of LSM_LOCK_UNLOCK, SHARED or EXCL */ - int bBlock /* True for a blocking lock */ -){ - lsm_db *pIter; - const u64 me = ((u64)1 << (iLock-1)); - const u64 ms = ((u64)1 << (iLock+32-1)); - int rc = LSM_OK; - Database *p = db->pDatabase; - - assert( eOp!=LSM_LOCK_EXCL || p->bReadonly==0 ); - assert( iLock>=1 && iLock<=LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1) ); - assert( LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1)<=32 ); - assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); - - /* Check for a no-op. Proceed only if this is not one of those. */ - if( (eOp==LSM_LOCK_UNLOCK && (db->mLock & (me|ms))!=0) - || (eOp==LSM_LOCK_SHARED && (db->mLock & (me|ms))!=ms) - || (eOp==LSM_LOCK_EXCL && (db->mLock & me)==0) - ){ - int nExcl = 0; /* Number of connections holding EXCLUSIVE */ - int nShared = 0; /* Number of connections holding SHARED */ - lsmMutexEnter(db->pEnv, p->pClientMutex); - - /* Figure out the locks currently held by this process on iLock, not - ** including any held by connection db. */ - for(pIter=p->pConn; pIter; pIter=pIter->pNext){ - assert( (pIter->mLock & me)==0 || (pIter->mLock & ms)!=0 ); - if( pIter!=db ){ - if( pIter->mLock & me ){ - nExcl++; - }else if( pIter->mLock & ms ){ - nShared++; - } - } - } - assert( nExcl==0 || nExcl==1 ); - assert( nExcl==0 || nShared==0 ); - assert( nExcl==0 || (db->mLock & (me|ms))==0 ); - - switch( eOp ){ - case LSM_LOCK_UNLOCK: - if( nShared==0 ){ - lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_UNLOCK); - } - db->mLock &= ~(me|ms); - break; - - case LSM_LOCK_SHARED: - if( nExcl ){ - rc = LSM_BUSY; - }else{ - if( nShared==0 ){ - rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_SHARED); - } - if( rc==LSM_OK ){ - db->mLock |= ms; - db->mLock &= ~me; - } - } - break; - - default: - assert( eOp==LSM_LOCK_EXCL ); - if( nExcl || nShared ){ - rc = LSM_BUSY; - }else{ - rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - db->mLock |= (me|ms); - } - } - break; - } - - lsmMutexLeave(db->pEnv, p->pClientMutex); - } - - return rc; -} - -#ifdef LSM_DEBUG - -int shmLockType(lsm_db *db, int iLock){ - const u64 me = ((u64)1 << (iLock-1)); - const u64 ms = ((u64)1 << (iLock+32-1)); - - if( db->mLock & me ) return LSM_LOCK_EXCL; - if( db->mLock & ms ) return LSM_LOCK_SHARED; - return LSM_LOCK_UNLOCK; -} - -/* -** The arguments passed to this function are similar to those passed to -** the lsmShmLock() function. However, instead of obtaining a new lock -** this function returns true if the specified connection already holds -** (or does not hold) such a lock, depending on the value of eOp. As -** follows: -** -** (eOp==LSM_LOCK_UNLOCK) -> true if db has no lock on iLock -** (eOp==LSM_LOCK_SHARED) -> true if db has at least a SHARED lock on iLock. -** (eOp==LSM_LOCK_EXCL) -> true if db has an EXCLUSIVE lock on iLock. -*/ -int lsmShmAssertLock(lsm_db *db, int iLock, int eOp){ - int ret = 0; - int eHave; - - assert( iLock>=1 && iLock<=LSM_LOCK_READER(LSM_LOCK_NREADER-1) ); - assert( iLock<=16 ); - assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); - - eHave = shmLockType(db, iLock); - - switch( eOp ){ - case LSM_LOCK_UNLOCK: - ret = (eHave==LSM_LOCK_UNLOCK); - break; - case LSM_LOCK_SHARED: - ret = (eHave!=LSM_LOCK_UNLOCK); - break; - case LSM_LOCK_EXCL: - ret = (eHave==LSM_LOCK_EXCL); - break; - default: - assert( !"bad eOp value passed to lsmShmAssertLock()" ); - break; - } - - return ret; -} - -int lsmShmAssertWorker(lsm_db *db){ - return lsmShmAssertLock(db, LSM_LOCK_WORKER, LSM_LOCK_EXCL) && db->pWorker; -} - -/* -** This function does not contribute to library functionality, and is not -** included in release builds. It is intended to be called from within -** an interactive debugger. -** -** When called, this function prints a single line of human readable output -** to stdout describing the locks currently held by the connection. For -** example: -** -** (gdb) call print_db_locks(pDb) -** (shared on dms2) (exclusive on writer) -*/ -void print_db_locks(lsm_db *db){ - int iLock; - for(iLock=0; iLock<16; iLock++){ - int bOne = 0; - const char *azLock[] = {0, "shared", "exclusive"}; - const char *azName[] = { - 0, "dms1", "dms2", "writer", "worker", "checkpointer", - "reader0", "reader1", "reader2", "reader3", "reader4", "reader5" - }; - int eHave = shmLockType(db, iLock); - if( azLock[eHave] ){ - printf("%s(%s on %s)", (bOne?" ":""), azLock[eHave], azName[iLock]); - bOne = 1; - } - } - printf("\n"); -} -void print_all_db_locks(lsm_db *db){ - lsm_db *p; - for(p=db->pDatabase->pConn; p; p=p->pNext){ - printf("%s connection %p ", ((p==db)?"*":""), p); - print_db_locks(p); - } -} -#endif - -void lsmShmBarrier(lsm_db *db){ - lsmEnvShmBarrier(db->pEnv); -} - -int lsm_checkpoint(lsm_db *pDb, int *pnKB){ - int rc; /* Return code */ - u32 nWrite = 0; /* Number of pages checkpointed */ - - /* Attempt the checkpoint. If successful, nWrite is set to the number of - ** pages written between this and the previous checkpoint. */ - rc = lsmCheckpointWrite(pDb, &nWrite); - - /* If required, calculate the output variable (KB of data checkpointed). - ** Set it to zero if an error occured. */ - if( pnKB ){ - int nKB = 0; - if( rc==LSM_OK && nWrite ){ - nKB = (((i64)nWrite * lsmFsPageSize(pDb->pFS)) + 1023) / 1024; - } - *pnKB = nKB; - } - - return rc; -} diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c deleted file mode 100644 index a72c8cafb..000000000 --- a/ext/lsm1/lsm_sorted.c +++ /dev/null @@ -1,6195 +0,0 @@ -/* -** 2011-08-14 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** PAGE FORMAT: -** -** The maximum page size is 65536 bytes. -** -** Since all records are equal to or larger than 2 bytes in size, and -** some space within the page is consumed by the page footer, there must -** be less than 2^15 records on each page. -** -** Each page ends with a footer that describes the pages contents. This -** footer serves as similar purpose to the page header in an SQLite database. -** A footer is used instead of a header because it makes it easier to -** populate a new page based on a sorted list of key/value pairs. -** -** The footer consists of the following values (starting at the end of -** the page and continuing backwards towards the start). All values are -** stored as unsigned big-endian integers. -** -** * Number of records on page (2 bytes). -** * Flags field (2 bytes). -** * Left-hand pointer value (8 bytes). -** * The starting offset of each record (2 bytes per record). -** -** Records may span pages. Unless it happens to be an exact fit, the part -** of the final record that starts on page X that does not fit on page X -** is stored at the start of page (X+1). This means there may be pages where -** (N==0). And on most pages the first record that starts on the page will -** not start at byte offset 0. For example: -** -** aaaaa bbbbb ccc <footer> cc eeeee fffff g <footer> gggg.... -** -** RECORD FORMAT: -** -** The first byte of the record is a flags byte. It is a combination -** of the following flags (defined in lsmInt.h): -** -** LSM_START_DELETE -** LSM_END_DELETE -** LSM_POINT_DELETE -** LSM_INSERT -** LSM_SEPARATOR -** LSM_SYSTEMKEY -** -** Immediately following the type byte is a pointer to the smallest key -** in the next file that is larger than the key in the current record. The -** pointer is encoded as a varint. When added to the 32-bit page number -** stored in the footer, it is the page number of the page that contains the -** smallest key in the next sorted file that is larger than this key. -** -** Next is the number of bytes in the key, encoded as a varint. -** -** If the LSM_INSERT flag is set, the number of bytes in the value, as -** a varint, is next. -** -** Finally, the blob of data containing the key, and for LSM_INSERT -** records, the value as well. -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -#define LSM_LOG_STRUCTURE 0 -#define LSM_LOG_DATA 0 - -/* -** Macros to help decode record types. -*/ -#define rtTopic(eType) ((eType) & LSM_SYSTEMKEY) -#define rtIsDelete(eType) (((eType) & 0x0F)==LSM_POINT_DELETE) - -#define rtIsSeparator(eType) (((eType) & LSM_SEPARATOR)!=0) -#define rtIsWrite(eType) (((eType) & LSM_INSERT)!=0) -#define rtIsSystem(eType) (((eType) & LSM_SYSTEMKEY)!=0) - -/* -** The following macros are used to access a page footer. -*/ -#define SEGMENT_NRECORD_OFFSET(pgsz) ((pgsz) - 2) -#define SEGMENT_FLAGS_OFFSET(pgsz) ((pgsz) - 2 - 2) -#define SEGMENT_POINTER_OFFSET(pgsz) ((pgsz) - 2 - 2 - 8) -#define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2) - -#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1) - -#define SEGMENT_BTREE_FLAG 0x0001 -#define PGFTR_SKIP_NEXT_FLAG 0x0002 -#define PGFTR_SKIP_THIS_FLAG 0x0004 - - -#ifndef LSM_SEGMENTPTR_FREE_THRESHOLD -# define LSM_SEGMENTPTR_FREE_THRESHOLD 1024 -#endif - -typedef struct SegmentPtr SegmentPtr; -typedef struct LsmBlob LsmBlob; - -struct LsmBlob { - lsm_env *pEnv; - void *pData; - int nData; - int nAlloc; -}; - -/* -** A SegmentPtr object may be used for one of two purposes: -** -** * To iterate and/or seek within a single Segment (the combination of a -** main run and an optional sorted run). -** -** * To iterate through the separators array of a segment. -*/ -struct SegmentPtr { - Level *pLevel; /* Level object segment is part of */ - Segment *pSeg; /* Segment to access */ - - /* Current page. See segmentPtrLoadPage(). */ - Page *pPg; /* Current page */ - u16 flags; /* Copy of page flags field */ - int nCell; /* Number of cells on pPg */ - LsmPgno iPtr; /* Base cascade pointer */ - - /* Current cell. See segmentPtrLoadCell() */ - int iCell; /* Current record within page pPg */ - int eType; /* Type of current record */ - LsmPgno iPgPtr; /* Cascade pointer offset */ - void *pKey; int nKey; /* Key associated with current record */ - void *pVal; int nVal; /* Current record value (eType==WRITE only) */ - - /* Blobs used to allocate buffers for pKey and pVal as required */ - LsmBlob blob1; - LsmBlob blob2; -}; - -/* -** Used to iterate through the keys stored in a b-tree hierarchy from start -** to finish. Only First() and Next() operations are required. -** -** btreeCursorNew() -** btreeCursorFirst() -** btreeCursorNext() -** btreeCursorFree() -** btreeCursorPosition() -** btreeCursorRestore() -*/ -typedef struct BtreePg BtreePg; -typedef struct BtreeCursor BtreeCursor; -struct BtreePg { - Page *pPage; - int iCell; -}; -struct BtreeCursor { - Segment *pSeg; /* Iterate through this segments btree */ - FileSystem *pFS; /* File system to read pages from */ - int nDepth; /* Allocated size of aPg[] */ - int iPg; /* Current entry in aPg[]. -1 -> EOF. */ - BtreePg *aPg; /* Pages from root to current location */ - - /* Cache of current entry. pKey==0 for EOF. */ - void *pKey; - int nKey; - int eType; - LsmPgno iPtr; - - /* Storage for key, if not local */ - LsmBlob blob; -}; - - -/* -** A cursor used for merged searches or iterations through up to one -** Tree structure and any number of sorted files. -** -** lsmMCursorNew() -** lsmMCursorSeek() -** lsmMCursorNext() -** lsmMCursorPrev() -** lsmMCursorFirst() -** lsmMCursorLast() -** lsmMCursorKey() -** lsmMCursorValue() -** lsmMCursorValid() -** -** iFree: -** This variable is only used by cursors providing input data for a -** new top-level segment. Such cursors only ever iterate forwards, not -** backwards. -*/ -struct MultiCursor { - lsm_db *pDb; /* Connection that owns this cursor */ - MultiCursor *pNext; /* Next cursor owned by connection pDb */ - int flags; /* Mask of CURSOR_XXX flags */ - - int eType; /* Cache of current key type */ - LsmBlob key; /* Cache of current key (or NULL) */ - LsmBlob val; /* Cache of current value */ - - /* All the component cursors: */ - TreeCursor *apTreeCsr[2]; /* Up to two tree cursors */ - int iFree; /* Next element of free-list (-ve for eof) */ - SegmentPtr *aPtr; /* Array of segment pointers */ - int nPtr; /* Size of array aPtr[] */ - BtreeCursor *pBtCsr; /* b-tree cursor (db writes only) */ - - /* Comparison results */ - int nTree; /* Size of aTree[] array */ - int *aTree; /* Array of comparison results */ - - /* Used by cursors flushing the in-memory tree only */ - void *pSystemVal; /* Pointer to buffer to free */ - - /* Used by worker cursors only */ - LsmPgno *pPrevMergePtr; -}; - -/* -** The following constants are used to assign integers to each component -** cursor of a multi-cursor. -*/ -#define CURSOR_DATA_TREE0 0 /* Current tree cursor (apTreeCsr[0]) */ -#define CURSOR_DATA_TREE1 1 /* The "old" tree, if any (apTreeCsr[1]) */ -#define CURSOR_DATA_SYSTEM 2 /* Free-list entries (new-toplevel only) */ -#define CURSOR_DATA_SEGMENT 3 /* First segment pointer (aPtr[0]) */ - -/* -** CURSOR_IGNORE_DELETE -** If set, this cursor will not visit SORTED_DELETE keys. -** -** CURSOR_FLUSH_FREELIST -** This cursor is being used to create a new toplevel. It should also -** iterate through the contents of the in-memory free block list. -** -** CURSOR_IGNORE_SYSTEM -** If set, this cursor ignores system keys. -** -** CURSOR_NEXT_OK -** Set if it is Ok to call lsm_csr_next(). -** -** CURSOR_PREV_OK -** Set if it is Ok to call lsm_csr_prev(). -** -** CURSOR_READ_SEPARATORS -** Set if this cursor should visit the separator keys in segment -** aPtr[nPtr-1]. -** -** CURSOR_SEEK_EQ -** Cursor has undergone a successful lsm_csr_seek(LSM_SEEK_EQ) operation. -** The key and value are stored in MultiCursor.key and MultiCursor.val -** respectively. -*/ -#define CURSOR_IGNORE_DELETE 0x00000001 -#define CURSOR_FLUSH_FREELIST 0x00000002 -#define CURSOR_IGNORE_SYSTEM 0x00000010 -#define CURSOR_NEXT_OK 0x00000020 -#define CURSOR_PREV_OK 0x00000040 -#define CURSOR_READ_SEPARATORS 0x00000080 -#define CURSOR_SEEK_EQ 0x00000100 - -typedef struct MergeWorker MergeWorker; -typedef struct Hierarchy Hierarchy; - -struct Hierarchy { - Page **apHier; - int nHier; -}; - -/* -** aSave: -** When mergeWorkerNextPage() is called to advance to the next page in -** the output segment, if the bStore flag for an element of aSave[] is -** true, it is cleared and the corresponding iPgno value is set to the -** page number of the page just completed. -** -** aSave[0] is used to record the pointer value to be pushed into the -** b-tree hierarchy. aSave[1] is used to save the page number of the -** page containing the indirect key most recently written to the b-tree. -** see mergeWorkerPushHierarchy() for details. -*/ -struct MergeWorker { - lsm_db *pDb; /* Database handle */ - Level *pLevel; /* Worker snapshot Level being merged */ - MultiCursor *pCsr; /* Cursor to read new segment contents from */ - int bFlush; /* True if this is an in-memory tree flush */ - Hierarchy hier; /* B-tree hierarchy under construction */ - Page *pPage; /* Current output page */ - int nWork; /* Number of calls to mergeWorkerNextPage() */ - LsmPgno *aGobble; /* Gobble point for each input segment */ - - LsmPgno iIndirect; - struct SavedPgno { - LsmPgno iPgno; - int bStore; - } aSave[2]; -}; - -#ifdef LSM_DEBUG_EXPENSIVE -static int assertPointersOk(lsm_db *, Segment *, Segment *, int); -static int assertBtreeOk(lsm_db *, Segment *); -static void assertRunInOrder(lsm_db *pDb, Segment *pSeg); -#else -#define assertRunInOrder(x,y) -#define assertBtreeOk(x,y) -#endif - - -struct FilePage { u8 *aData; int nData; }; -static u8 *fsPageData(Page *pPg, int *pnData){ - *pnData = ((struct FilePage *)(pPg))->nData; - return ((struct FilePage *)(pPg))->aData; -} -/*UNUSED static u8 *fsPageDataPtr(Page *pPg){ - return ((struct FilePage *)(pPg))->aData; -}*/ - -/* -** Write nVal as a 16-bit unsigned big-endian integer into buffer aOut. -*/ -void lsmPutU16(u8 *aOut, u16 nVal){ - aOut[0] = (u8)((nVal>>8) & 0xFF); - aOut[1] = (u8)(nVal & 0xFF); -} - -void lsmPutU32(u8 *aOut, u32 nVal){ - aOut[0] = (u8)((nVal>>24) & 0xFF); - aOut[1] = (u8)((nVal>>16) & 0xFF); - aOut[2] = (u8)((nVal>> 8) & 0xFF); - aOut[3] = (u8)((nVal ) & 0xFF); -} - -int lsmGetU16(u8 *aOut){ - return (aOut[0] << 8) + aOut[1]; -} - -u32 lsmGetU32(u8 *aOut){ - return ((u32)aOut[0] << 24) - + ((u32)aOut[1] << 16) - + ((u32)aOut[2] << 8) - + ((u32)aOut[3]); -} - -u64 lsmGetU64(u8 *aOut){ - return ((u64)aOut[0] << 56) - + ((u64)aOut[1] << 48) - + ((u64)aOut[2] << 40) - + ((u64)aOut[3] << 32) - + ((u64)aOut[4] << 24) - + ((u32)aOut[5] << 16) - + ((u32)aOut[6] << 8) - + ((u32)aOut[7]); -} - -void lsmPutU64(u8 *aOut, u64 nVal){ - aOut[0] = (u8)((nVal>>56) & 0xFF); - aOut[1] = (u8)((nVal>>48) & 0xFF); - aOut[2] = (u8)((nVal>>40) & 0xFF); - aOut[3] = (u8)((nVal>>32) & 0xFF); - aOut[4] = (u8)((nVal>>24) & 0xFF); - aOut[5] = (u8)((nVal>>16) & 0xFF); - aOut[6] = (u8)((nVal>> 8) & 0xFF); - aOut[7] = (u8)((nVal ) & 0xFF); -} - -static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){ - assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) ); - if( pBlob->nAlloc<nData ){ - pBlob->pData = lsmReallocOrFree(pEnv, pBlob->pData, nData); - if( !pBlob->pData ) return LSM_NOMEM_BKPT; - pBlob->nAlloc = nData; - pBlob->pEnv = pEnv; - } - return LSM_OK; -} - -static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){ - if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM; - memcpy(pBlob->pData, pData, nData); - pBlob->nData = nData; - return LSM_OK; -} - -#if 0 -static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){ - return sortedBlobSet(pDest, pSrc->pData, pSrc->nData); -} -#endif - -static void sortedBlobFree(LsmBlob *pBlob){ - assert( pBlob->pEnv || pBlob->pData==0 ); - if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData); - memset(pBlob, 0, sizeof(LsmBlob)); -} - -static int sortedReadData( - Segment *pSeg, - Page *pPg, - int iOff, - int nByte, - void **ppData, - LsmBlob *pBlob -){ - int rc = LSM_OK; - int iEnd; - int nData; - int nCell; - u8 *aData; - - aData = fsPageData(pPg, &nData); - nCell = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); - iEnd = SEGMENT_EOF(nData, nCell); - assert( iEnd>0 && iEnd<nData ); - - if( iOff+nByte<=iEnd ){ - *ppData = (void *)&aData[iOff]; - }else{ - int nRem = nByte; - int i = iOff; - u8 *aDest; - - /* Make sure the blob is big enough to store the value being loaded. */ - rc = sortedBlobGrow(lsmPageEnv(pPg), pBlob, nByte); - if( rc!=LSM_OK ) return rc; - pBlob->nData = nByte; - aDest = (u8 *)pBlob->pData; - *ppData = pBlob->pData; - - /* Increment the pointer pages ref-count. */ - lsmFsPageRef(pPg); - - while( rc==LSM_OK ){ - Page *pNext; - int flags; - - /* Copy data from pPg into the output buffer. */ - int nCopy = LSM_MIN(nRem, iEnd-i); - if( nCopy>0 ){ - memcpy(&aDest[nByte-nRem], &aData[i], nCopy); - nRem -= nCopy; - i += nCopy; - assert( nRem==0 || i==iEnd ); - } - assert( nRem>=0 ); - if( nRem==0 ) break; - i -= iEnd; - - /* Grab the next page in the segment */ - - do { - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - if( rc==LSM_OK && pNext==0 ){ - rc = LSM_CORRUPT_BKPT; - } - if( rc ) break; - lsmFsPageRelease(pPg); - pPg = pNext; - aData = fsPageData(pPg, &nData); - flags = lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]); - }while( flags&SEGMENT_BTREE_FLAG ); - - iEnd = SEGMENT_EOF(nData, lsmGetU16(&aData[nData-2])); - assert( iEnd>0 && iEnd<nData ); - } - - lsmFsPageRelease(pPg); - } - - return rc; -} - -static int pageGetNRec(u8 *aData, int nData){ - return (int)lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); -} - -static LsmPgno pageGetPtr(u8 *aData, int nData){ - return (LsmPgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]); -} - -static int pageGetFlags(u8 *aData, int nData){ - return (int)lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]); -} - -static u8 *pageGetCell(u8 *aData, int nData, int iCell){ - return &aData[lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, iCell)])]; -} - -/* -** Return the number of cells on page pPg. -*/ -static int pageObjGetNRec(Page *pPg){ - int nData; - u8 *aData = lsmFsPageData(pPg, &nData); - return pageGetNRec(aData, nData); -} - -/* -** Return the decoded (possibly relative) pointer value stored in cell -** iCell from page aData/nData. -*/ -static LsmPgno pageGetRecordPtr(u8 *aData, int nData, int iCell){ - LsmPgno iRet; /* Return value */ - u8 *aCell; /* Pointer to cell iCell */ - - assert( iCell<pageGetNRec(aData, nData) && iCell>=0 ); - aCell = pageGetCell(aData, nData, iCell); - lsmVarintGet64(&aCell[1], &iRet); - return iRet; -} - -static u8 *pageGetKey( - Segment *pSeg, /* Segment pPg belongs to */ - Page *pPg, /* Page to read from */ - int iCell, /* Index of cell on page to read */ - int *piTopic, /* OUT: Topic associated with this key */ - int *pnKey, /* OUT: Size of key in bytes */ - LsmBlob *pBlob /* If required, use this for dynamic memory */ -){ - u8 *pKey; - i64 nDummy; - int eType; - u8 *aData; - int nData; - - aData = fsPageData(pPg, &nData); - - assert( !(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ); - assert( iCell<pageGetNRec(aData, nData) ); - - pKey = pageGetCell(aData, nData, iCell); - eType = *pKey++; - pKey += lsmVarintGet64(pKey, &nDummy); - pKey += lsmVarintGet32(pKey, pnKey); - if( rtIsWrite(eType) ){ - pKey += lsmVarintGet64(pKey, &nDummy); - } - *piTopic = rtTopic(eType); - - sortedReadData(pSeg, pPg, pKey-aData, *pnKey, (void **)&pKey, pBlob); - return pKey; -} - -static int pageGetKeyCopy( - lsm_env *pEnv, /* Environment handle */ - Segment *pSeg, /* Segment pPg belongs to */ - Page *pPg, /* Page to read from */ - int iCell, /* Index of cell on page to read */ - int *piTopic, /* OUT: Topic associated with this key */ - LsmBlob *pBlob /* If required, use this for dynamic memory */ -){ - int rc = LSM_OK; - int nKey; - u8 *aKey; - - aKey = pageGetKey(pSeg, pPg, iCell, piTopic, &nKey, pBlob); - assert( (void *)aKey!=pBlob->pData || nKey==pBlob->nData ); - if( (void *)aKey!=pBlob->pData ){ - rc = sortedBlobSet(pEnv, pBlob, aKey, nKey); - } - - return rc; -} - -static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){ - LsmPgno iRef; - u8 *aData; - int nData; - u8 *aCell; - - aData = fsPageData(pPg, &nData); - aCell = pageGetCell(aData, nData, iKey); - assert( aCell[0]==0 ); - aCell++; - aCell += lsmVarintGet64(aCell, &iRef); - lsmVarintGet64(aCell, &iRef); - assert( iRef>0 ); - return iRef; -} - -#define GETVARINT64(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet64((a), &(i))) -#define GETVARINT32(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet32((a), &(i))) - -static int pageGetBtreeKey( - Segment *pSeg, /* Segment page pPg belongs to */ - Page *pPg, - int iKey, - LsmPgno *piPtr, - int *piTopic, - void **ppKey, - int *pnKey, - LsmBlob *pBlob -){ - u8 *aData; - int nData; - u8 *aCell; - int eType; - - aData = fsPageData(pPg, &nData); - assert( SEGMENT_BTREE_FLAG & pageGetFlags(aData, nData) ); - assert( iKey>=0 && iKey<pageGetNRec(aData, nData) ); - - aCell = pageGetCell(aData, nData, iKey); - eType = *aCell++; - aCell += GETVARINT64(aCell, *piPtr); - - if( eType==0 ){ - int rc; - LsmPgno iRef; /* Page number of referenced page */ - Page *pRef; - aCell += GETVARINT64(aCell, iRef); - rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef); - if( rc!=LSM_OK ) return rc; - pageGetKeyCopy(lsmPageEnv(pPg), pSeg, pRef, 0, &eType, pBlob); - lsmFsPageRelease(pRef); - *ppKey = pBlob->pData; - *pnKey = pBlob->nData; - }else{ - aCell += GETVARINT32(aCell, *pnKey); - *ppKey = aCell; - } - if( piTopic ) *piTopic = rtTopic(eType); - - return LSM_OK; -} - -static int btreeCursorLoadKey(BtreeCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->iPg<0 ){ - pCsr->pKey = 0; - pCsr->nKey = 0; - pCsr->eType = 0; - }else{ - LsmPgno dummy; - int iPg = pCsr->iPg; - int iCell = pCsr->aPg[iPg].iCell; - while( iCell<0 && (--iPg)>=0 ){ - iCell = pCsr->aPg[iPg].iCell-1; - } - if( iPg<0 || iCell<0 ) return LSM_CORRUPT_BKPT; - - rc = pageGetBtreeKey( - pCsr->pSeg, - pCsr->aPg[iPg].pPage, iCell, - &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob - ); - pCsr->eType |= LSM_SEPARATOR; - } - - return rc; -} - -static LsmPgno btreeCursorPtr(u8 *aData, int nData, int iCell){ - int nCell; - - nCell = pageGetNRec(aData, nData); - if( iCell>=nCell ){ - return pageGetPtr(aData, nData); - } - return pageGetRecordPtr(aData, nData, iCell); -} - -static int btreeCursorNext(BtreeCursor *pCsr){ - int rc = LSM_OK; - - BtreePg *pPg = &pCsr->aPg[pCsr->iPg]; - int nCell; - u8 *aData; - int nData; - - assert( pCsr->iPg>=0 ); - assert( pCsr->iPg==pCsr->nDepth-1 ); - - aData = fsPageData(pPg->pPage, &nData); - nCell = pageGetNRec(aData, nData); - assert( pPg->iCell<=nCell ); - pPg->iCell++; - if( pPg->iCell==nCell ){ - LsmPgno iLoad; - - /* Up to parent. */ - lsmFsPageRelease(pPg->pPage); - pPg->pPage = 0; - pCsr->iPg--; - while( pCsr->iPg>=0 ){ - pPg = &pCsr->aPg[pCsr->iPg]; - aData = fsPageData(pPg->pPage, &nData); - if( pPg->iCell<pageGetNRec(aData, nData) ) break; - lsmFsPageRelease(pPg->pPage); - pCsr->iPg--; - } - - /* Read the key */ - rc = btreeCursorLoadKey(pCsr); - - /* Unless the cursor is at EOF, descend to cell -1 (yes, negative one) of - ** the left-most most descendent. */ - if( pCsr->iPg>=0 ){ - pCsr->aPg[pCsr->iPg].iCell++; - - iLoad = btreeCursorPtr(aData, nData, pPg->iCell); - do { - Page *pLoad; - pCsr->iPg++; - rc = lsmFsDbPageGet(pCsr->pFS, pCsr->pSeg, iLoad, &pLoad); - pCsr->aPg[pCsr->iPg].pPage = pLoad; - pCsr->aPg[pCsr->iPg].iCell = 0; - if( rc==LSM_OK ){ - if( pCsr->iPg==(pCsr->nDepth-1) ) break; - aData = fsPageData(pLoad, &nData); - iLoad = btreeCursorPtr(aData, nData, 0); - } - }while( rc==LSM_OK && pCsr->iPg<(pCsr->nDepth-1) ); - pCsr->aPg[pCsr->iPg].iCell = -1; - } - - }else{ - rc = btreeCursorLoadKey(pCsr); - } - - if( rc==LSM_OK && pCsr->iPg>=0 ){ - aData = fsPageData(pCsr->aPg[pCsr->iPg].pPage, &nData); - pCsr->iPtr = btreeCursorPtr(aData, nData, pCsr->aPg[pCsr->iPg].iCell+1); - } - - return rc; -} - -static void btreeCursorFree(BtreeCursor *pCsr){ - if( pCsr ){ - int i; - lsm_env *pEnv = lsmFsEnv(pCsr->pFS); - for(i=0; i<=pCsr->iPg; i++){ - lsmFsPageRelease(pCsr->aPg[i].pPage); - } - sortedBlobFree(&pCsr->blob); - lsmFree(pEnv, pCsr->aPg); - lsmFree(pEnv, pCsr); - } -} - -static int btreeCursorFirst(BtreeCursor *pCsr){ - int rc; - - Page *pPg = 0; - FileSystem *pFS = pCsr->pFS; - LsmPgno iPg = pCsr->pSeg->iRoot; - - do { - rc = lsmFsDbPageGet(pFS, pCsr->pSeg, iPg, &pPg); - assert( (rc==LSM_OK)==(pPg!=0) ); - if( rc==LSM_OK ){ - u8 *aData; - int nData; - int flags; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( (flags & SEGMENT_BTREE_FLAG)==0 ) break; - - if( (pCsr->nDepth % 8)==0 ){ - int nNew = pCsr->nDepth + 8; - pCsr->aPg = (BtreePg *)lsmReallocOrFreeRc( - lsmFsEnv(pFS), pCsr->aPg, sizeof(BtreePg) * nNew, &rc - ); - if( rc==LSM_OK ){ - memset(&pCsr->aPg[pCsr->nDepth], 0, sizeof(BtreePg) * 8); - } - } - - if( rc==LSM_OK ){ - assert( pCsr->aPg[pCsr->nDepth].iCell==0 ); - pCsr->aPg[pCsr->nDepth].pPage = pPg; - pCsr->nDepth++; - iPg = pageGetRecordPtr(aData, nData, 0); - } - } - }while( rc==LSM_OK ); - lsmFsPageRelease(pPg); - pCsr->iPg = pCsr->nDepth-1; - - if( rc==LSM_OK && pCsr->nDepth ){ - pCsr->aPg[pCsr->iPg].iCell = -1; - rc = btreeCursorNext(pCsr); - } - - return rc; -} - -static void btreeCursorPosition(BtreeCursor *pCsr, MergeInput *p){ - if( pCsr->iPg>=0 ){ - p->iPg = lsmFsPageNumber(pCsr->aPg[pCsr->iPg].pPage); - p->iCell = ((pCsr->aPg[pCsr->iPg].iCell + 1) << 8) + pCsr->nDepth; - }else{ - p->iPg = 0; - p->iCell = 0; - } -} - -static void btreeCursorSplitkey(BtreeCursor *pCsr, MergeInput *p){ - int iCell = pCsr->aPg[pCsr->iPg].iCell; - if( iCell>=0 ){ - p->iCell = iCell; - p->iPg = lsmFsPageNumber(pCsr->aPg[pCsr->iPg].pPage); - }else{ - int i; - for(i=pCsr->iPg-1; i>=0; i--){ - if( pCsr->aPg[i].iCell>0 ) break; - } - assert( i>=0 ); - p->iCell = pCsr->aPg[i].iCell-1; - p->iPg = lsmFsPageNumber(pCsr->aPg[i].pPage); - } -} - -static int sortedKeyCompare( - int (*xCmp)(void *, int, void *, int), - int iLhsTopic, void *pLhsKey, int nLhsKey, - int iRhsTopic, void *pRhsKey, int nRhsKey -){ - int res = iLhsTopic - iRhsTopic; - if( res==0 ){ - res = xCmp(pLhsKey, nLhsKey, pRhsKey, nRhsKey); - } - return res; -} - -static int btreeCursorRestore( - BtreeCursor *pCsr, - int (*xCmp)(void *, int, void *, int), - MergeInput *p -){ - int rc = LSM_OK; - - if( p->iPg ){ - lsm_env *pEnv = lsmFsEnv(pCsr->pFS); - int iCell; /* Current cell number on leaf page */ - LsmPgno iLeaf; /* Page number of current leaf page */ - int nDepth; /* Depth of b-tree structure */ - Segment *pSeg = pCsr->pSeg; - - /* Decode the MergeInput structure */ - iLeaf = p->iPg; - nDepth = (p->iCell & 0x00FF); - iCell = (p->iCell >> 8) - 1; - - /* Allocate the BtreeCursor.aPg[] array */ - assert( pCsr->aPg==0 ); - pCsr->aPg = (BtreePg *)lsmMallocZeroRc(pEnv, sizeof(BtreePg) * nDepth, &rc); - - /* Populate the last entry of the aPg[] array */ - if( rc==LSM_OK ){ - Page **pp = &pCsr->aPg[nDepth-1].pPage; - pCsr->iPg = nDepth-1; - pCsr->nDepth = nDepth; - pCsr->aPg[pCsr->iPg].iCell = iCell; - rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp); - } - - /* Populate any other aPg[] array entries */ - if( rc==LSM_OK && nDepth>1 ){ - LsmBlob blob = {0,0,0}; - void *pSeek; - int nSeek; - int iTopicSeek; - int iPg = 0; - LsmPgno iLoad = pSeg->iRoot; - Page *pPg = pCsr->aPg[nDepth-1].pPage; - - if( pageObjGetNRec(pPg)==0 ){ - /* This can happen when pPg is the right-most leaf in the b-tree. - ** In this case, set the iTopicSeek/pSeek/nSeek key to a value - ** greater than any real key. */ - assert( iCell==-1 ); - iTopicSeek = 1000; - pSeek = 0; - nSeek = 0; - }else{ - LsmPgno dummy; - rc = pageGetBtreeKey(pSeg, pPg, - 0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob - ); - } - - do { - Page *pPg2; - rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLoad, &pPg2); - assert( rc==LSM_OK || pPg2==0 ); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer containing page data */ - int nData; /* Size of aData[] in bytes */ - int iMin; - int iMax; - int iCell2; - - aData = fsPageData(pPg2, &nData); - assert( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ); - - iLoad = pageGetPtr(aData, nData); - iCell2 = pageGetNRec(aData, nData); - iMax = iCell2-1; - iMin = 0; - - while( iMax>=iMin ){ - int iTry = (iMin+iMax)/2; - void *pKey; int nKey; /* Key for cell iTry */ - int iTopic; /* Topic for key pKeyT/nKeyT */ - LsmPgno iPtr; /* Pointer for cell iTry */ - int res; /* (pSeek - pKeyT) */ - - rc = pageGetBtreeKey( - pSeg, pPg2, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob - ); - if( rc!=LSM_OK ) break; - - res = sortedKeyCompare( - xCmp, iTopicSeek, pSeek, nSeek, iTopic, pKey, nKey - ); - assert( res!=0 ); - - if( res<0 ){ - iLoad = iPtr; - iCell2 = iTry; - iMax = iTry-1; - }else{ - iMin = iTry+1; - } - } - - pCsr->aPg[iPg].pPage = pPg2; - pCsr->aPg[iPg].iCell = iCell2; - iPg++; - assert( iPg!=nDepth-1 - || lsmFsRedirectPage(pCsr->pFS, pSeg->pRedirect, iLoad)==iLeaf - ); - } - }while( rc==LSM_OK && iPg<(nDepth-1) ); - sortedBlobFree(&blob); - } - - /* Load the current key and pointer */ - if( rc==LSM_OK ){ - BtreePg *pBtreePg; - u8 *aData; - int nData; - - pBtreePg = &pCsr->aPg[pCsr->iPg]; - aData = fsPageData(pBtreePg->pPage, &nData); - pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1); - if( pBtreePg->iCell<0 ){ - LsmPgno dummy; - int i; - for(i=pCsr->iPg-1; i>=0; i--){ - if( pCsr->aPg[i].iCell>0 ) break; - } - assert( i>=0 ); - rc = pageGetBtreeKey(pSeg, - pCsr->aPg[i].pPage, pCsr->aPg[i].iCell-1, - &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob - ); - pCsr->eType |= LSM_SEPARATOR; - - }else{ - rc = btreeCursorLoadKey(pCsr); - } - } - } - return rc; -} - -static int btreeCursorNew( - lsm_db *pDb, - Segment *pSeg, - BtreeCursor **ppCsr -){ - int rc = LSM_OK; - BtreeCursor *pCsr; - - assert( pSeg->iRoot ); - pCsr = lsmMallocZeroRc(pDb->pEnv, sizeof(BtreeCursor), &rc); - if( pCsr ){ - pCsr->pFS = pDb->pFS; - pCsr->pSeg = pSeg; - pCsr->iPg = -1; - } - - *ppCsr = pCsr; - return rc; -} - -static void segmentPtrSetPage(SegmentPtr *pPtr, Page *pNext){ - lsmFsPageRelease(pPtr->pPg); - if( pNext ){ - int nData; - u8 *aData = fsPageData(pNext, &nData); - pPtr->nCell = pageGetNRec(aData, nData); - pPtr->flags = (u16)pageGetFlags(aData, nData); - pPtr->iPtr = pageGetPtr(aData, nData); - } - pPtr->pPg = pNext; -} - -/* -** Load a new page into the SegmentPtr object pPtr. -*/ -static int segmentPtrLoadPage( - FileSystem *pFS, - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - LsmPgno iNew /* Page number of new page */ -){ - Page *pPg = 0; /* The new page */ - int rc; /* Return Code */ - - rc = lsmFsDbPageGet(pFS, pPtr->pSeg, iNew, &pPg); - assert( rc==LSM_OK || pPg==0 ); - segmentPtrSetPage(pPtr, pPg); - - return rc; -} - -static int segmentPtrReadData( - SegmentPtr *pPtr, - int iOff, - int nByte, - void **ppData, - LsmBlob *pBlob -){ - return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob); -} - -static int segmentPtrNextPage( - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - int eDir /* +1 for next(), -1 for prev() */ -){ - Page *pNext; /* New page to load */ - int rc; /* Return code */ - - assert( eDir==1 || eDir==-1 ); - assert( pPtr->pPg ); - assert( pPtr->pSeg || eDir>0 ); - - rc = lsmFsDbPageNext(pPtr->pSeg, pPtr->pPg, eDir, &pNext); - assert( rc==LSM_OK || pNext==0 ); - segmentPtrSetPage(pPtr, pNext); - return rc; -} - -static int segmentPtrLoadCell( - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - int iNew /* Cell number of new cell */ -){ - int rc = LSM_OK; - if( pPtr->pPg ){ - u8 *aData; /* Pointer to page data buffer */ - int iOff; /* Offset in aData[] to read from */ - int nPgsz; /* Size of page (aData[]) in bytes */ - - assert( iNew<pPtr->nCell ); - pPtr->iCell = iNew; - aData = fsPageData(pPtr->pPg, &nPgsz); - iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nPgsz, pPtr->iCell)]); - pPtr->eType = aData[iOff]; - iOff++; - iOff += GETVARINT64(&aData[iOff], pPtr->iPgPtr); - iOff += GETVARINT32(&aData[iOff], pPtr->nKey); - if( rtIsWrite(pPtr->eType) ){ - iOff += GETVARINT32(&aData[iOff], pPtr->nVal); - } - assert( pPtr->nKey>=0 ); - - rc = segmentPtrReadData( - pPtr, iOff, pPtr->nKey, &pPtr->pKey, &pPtr->blob1 - ); - if( rc==LSM_OK && rtIsWrite(pPtr->eType) ){ - rc = segmentPtrReadData( - pPtr, iOff+pPtr->nKey, pPtr->nVal, &pPtr->pVal, &pPtr->blob2 - ); - }else{ - pPtr->nVal = 0; - pPtr->pVal = 0; - } - } - - return rc; -} - - -static Segment *sortedSplitkeySegment(Level *pLevel){ - Merge *pMerge = pLevel->pMerge; - MergeInput *p = &pMerge->splitkey; - Segment *pSeg; - int i; - - for(i=0; i<pMerge->nInput; i++){ - if( p->iPg==pMerge->aInput[i].iPg ) break; - } - if( pMerge->nInput==(pLevel->nRight+1) && i>=(pMerge->nInput-1) ){ - pSeg = &pLevel->pNext->lhs; - }else{ - pSeg = &pLevel->aRhs[i]; - } - - return pSeg; -} - -static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){ - Segment *pSeg; - Page *pPg = 0; - lsm_env *pEnv = pDb->pEnv; /* Environment handle */ - int rc = *pRc; - Merge *pMerge = pLevel->pMerge; - - pSeg = sortedSplitkeySegment(pLevel); - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg); - } - if( rc==LSM_OK ){ - int iTopic; - LsmBlob blob = {0, 0, 0, 0}; - u8 *aData; - int nData; - - aData = lsmFsPageData(pPg, &nData); - if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){ - void *pKey; - int nKey; - LsmPgno dummy; - rc = pageGetBtreeKey(pSeg, - pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob - ); - if( rc==LSM_OK && blob.pData!=pKey ){ - rc = sortedBlobSet(pEnv, &blob, pKey, nKey); - } - }else{ - rc = pageGetKeyCopy( - pEnv, pSeg, pPg, pMerge->splitkey.iCell, &iTopic, &blob - ); - } - - pLevel->iSplitTopic = iTopic; - pLevel->pSplitKey = blob.pData; - pLevel->nSplitKey = blob.nData; - lsmFsPageRelease(pPg); - } - - *pRc = rc; -} - -/* -** Reset a segment cursor. Also free its buffers if they are nThreshold -** bytes or larger in size. -*/ -static void segmentPtrReset(SegmentPtr *pPtr, int nThreshold){ - lsmFsPageRelease(pPtr->pPg); - pPtr->pPg = 0; - pPtr->nCell = 0; - pPtr->pKey = 0; - pPtr->nKey = 0; - pPtr->pVal = 0; - pPtr->nVal = 0; - pPtr->eType = 0; - pPtr->iCell = 0; - if( pPtr->blob1.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob1); - if( pPtr->blob2.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob2); -} - -static int segmentPtrIgnoreSeparators(MultiCursor *pCsr, SegmentPtr *pPtr){ - return (pCsr->flags & CURSOR_READ_SEPARATORS)==0 - || (pPtr!=&pCsr->aPtr[pCsr->nPtr-1]); -} - -static int segmentPtrAdvance( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int bReverse -){ - int eDir = (bReverse ? -1 : 1); - Level *pLvl = pPtr->pLevel; - do { - int rc; - int iCell; /* Number of new cell in page */ - int svFlags = 0; /* SegmentPtr.eType before advance */ - - iCell = pPtr->iCell + eDir; - assert( pPtr->pPg ); - assert( iCell<=pPtr->nCell && iCell>=-1 ); - - if( bReverse && pPtr->pSeg!=&pPtr->pLevel->lhs ){ - svFlags = pPtr->eType; - assert( svFlags ); - } - - if( iCell>=pPtr->nCell || iCell<0 ){ - do { - rc = segmentPtrNextPage(pPtr, eDir); - }while( rc==LSM_OK - && pPtr->pPg - && (pPtr->nCell==0 || (pPtr->flags & SEGMENT_BTREE_FLAG) ) - ); - if( rc!=LSM_OK ) return rc; - iCell = bReverse ? (pPtr->nCell-1) : 0; - } - rc = segmentPtrLoadCell(pPtr, iCell); - if( rc!=LSM_OK ) return rc; - - if( svFlags && pPtr->pPg ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - - if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){ - Segment *pSeg = pPtr->pSeg; - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg); - if( rc!=LSM_OK ) return rc; - pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE; - pPtr->eType |= (pLvl->iSplitTopic ? LSM_SYSTEMKEY : 0); - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - } - - }while( pCsr - && pPtr->pPg - && segmentPtrIgnoreSeparators(pCsr, pPtr) - && rtIsSeparator(pPtr->eType) - ); - - return LSM_OK; -} - -static void segmentPtrEndPage( - FileSystem *pFS, - SegmentPtr *pPtr, - int bLast, - int *pRc -){ - if( *pRc==LSM_OK ){ - Segment *pSeg = pPtr->pSeg; - Page *pNew = 0; - if( bLast ){ - *pRc = lsmFsDbPageLast(pFS, pSeg, &pNew); - }else{ - *pRc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pNew); - } - segmentPtrSetPage(pPtr, pNew); - } -} - - -/* -** Try to move the segment pointer passed as the second argument so that it -** points at either the first (bLast==0) or last (bLast==1) cell in the valid -** region of the segment defined by pPtr->iFirst and pPtr->iLast. -** -** Return LSM_OK if successful or an lsm error code if something goes -** wrong (IO error, OOM etc.). -*/ -static int segmentPtrEnd(MultiCursor *pCsr, SegmentPtr *pPtr, int bLast){ - Level *pLvl = pPtr->pLevel; - int rc = LSM_OK; - FileSystem *pFS = pCsr->pDb->pFS; - int bIgnore; - - segmentPtrEndPage(pFS, pPtr, bLast, &rc); - while( rc==LSM_OK && pPtr->pPg - && (pPtr->nCell==0 || (pPtr->flags & SEGMENT_BTREE_FLAG)) - ){ - rc = segmentPtrNextPage(pPtr, (bLast ? -1 : 1)); - } - - if( rc==LSM_OK && pPtr->pPg ){ - rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); - if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - - bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr); - if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){ - rc = segmentPtrAdvance(pCsr, pPtr, bLast); - } - -#if 0 - if( bLast && rc==LSM_OK && pPtr->pPg - && pPtr->pSeg==&pLvl->lhs - && pLvl->nRight && (pPtr->eType & LSM_START_DELETE) - ){ - pPtr->iCell++; - pPtr->eType = LSM_END_DELETE | (pLvl->iSplitTopic); - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - pPtr->pVal = 0; - pPtr->nVal = 0; - } -#endif - - return rc; -} - -static void segmentPtrKey(SegmentPtr *pPtr, void **ppKey, int *pnKey){ - assert( pPtr->pPg ); - *ppKey = pPtr->pKey; - *pnKey = pPtr->nKey; -} - -#if 0 /* NOT USED */ -static char *keyToString(lsm_env *pEnv, void *pKey, int nKey){ - int i; - u8 *aKey = (u8 *)pKey; - char *zRet = (char *)lsmMalloc(pEnv, nKey+1); - - for(i=0; i<nKey; i++){ - zRet[i] = (char)(isalnum(aKey[i]) ? aKey[i] : '.'); - } - zRet[nKey] = '\0'; - return zRet; -} -#endif - -#if 0 /* NOT USED */ -/* -** Check that the page that pPtr currently has loaded is the correct page -** to search for key (pKey/nKey). If it is, return 1. Otherwise, an assert -** fails and this function does not return. -*/ -static int assertKeyLocation( - MultiCursor *pCsr, - SegmentPtr *pPtr, - void *pKey, int nKey -){ - lsm_env *pEnv = lsmFsEnv(pCsr->pDb->pFS); - LsmBlob blob = {0, 0, 0}; - int eDir; - int iTopic = 0; /* TODO: Fix me */ - - for(eDir=-1; eDir<=1; eDir+=2){ - Page *pTest = pPtr->pPg; - - lsmFsPageRef(pTest); - while( pTest ){ - Segment *pSeg = pPtr->pSeg; - Page *pNext; - - int rc = lsmFsDbPageNext(pSeg, pTest, eDir, &pNext); - lsmFsPageRelease(pTest); - if( rc ) return 1; - pTest = pNext; - - if( pTest ){ - int nData; - u8 *aData = fsPageData(pTest, &nData); - int nCell = pageGetNRec(aData, nData); - int flags = pageGetFlags(aData, nData); - if( nCell && 0==(flags&SEGMENT_BTREE_FLAG) ){ - int nPgKey; - int iPgTopic; - u8 *pPgKey; - int res; - int iCell; - - iCell = ((eDir < 0) ? (nCell-1) : 0); - pPgKey = pageGetKey(pSeg, pTest, iCell, &iPgTopic, &nPgKey, &blob); - res = iTopic - iPgTopic; - if( res==0 ) res = pCsr->pDb->xCmp(pKey, nKey, pPgKey, nPgKey); - if( (eDir==1 && res>0) || (eDir==-1 && res<0) ){ - /* Taking this branch means something has gone wrong. */ - char *zMsg = lsmMallocPrintf(pEnv, "Key \"%s\" is not on page %d", - keyToString(pEnv, pKey, nKey), lsmFsPageNumber(pPtr->pPg) - ); - fprintf(stderr, "%s\n", zMsg); - assert( !"assertKeyLocation() failed" ); - } - lsmFsPageRelease(pTest); - pTest = 0; - } - } - } - } - - sortedBlobFree(&blob); - return 1; -} -#endif - -#ifndef NDEBUG -static int assertSeekResult( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int iTopic, - void *pKey, - int nKey, - int eSeek -){ - if( pPtr->pPg ){ - int res; - res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey - ); - - if( eSeek==LSM_SEEK_EQ ) return (res==0); - if( eSeek==LSM_SEEK_LE ) return (res>=0); - if( eSeek==LSM_SEEK_GE ) return (res<=0); - } - - return 1; -} -#endif - -static int segmentPtrSearchOversized( - MultiCursor *pCsr, /* Cursor context */ - SegmentPtr *pPtr, /* Pointer to seek */ - int iTopic, /* Topic of key to search for */ - void *pKey, int nKey /* Key to seek to */ -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int rc = LSM_OK; - - /* If the OVERSIZED flag is set, then there is no pointer in the - ** upper level to the next page in the segment that contains at least - ** one key. So compare the largest key on the current page with the - ** key being sought (pKey/nKey). If (pKey/nKey) is larger, advance - ** to the next page in the segment that contains at least one key. - */ - while( rc==LSM_OK && (pPtr->flags & PGFTR_SKIP_NEXT_FLAG) ){ - u8 *pLastKey; - int nLastKey; - int iLastTopic; - int res; /* Result of comparison */ - Page *pNext; - - /* Load the last key on the current page. */ - pLastKey = pageGetKey(pPtr->pSeg, - pPtr->pPg, pPtr->nCell-1, &iLastTopic, &nLastKey, &pPtr->blob1 - ); - - /* If the loaded key is >= than (pKey/nKey), break out of the loop. - ** If (pKey/nKey) is present in this array, it must be on the current - ** page. */ - res = sortedKeyCompare( - xCmp, iLastTopic, pLastKey, nLastKey, iTopic, pKey, nKey - ); - if( res>=0 ) break; - - /* Advance to the next page that contains at least one key. */ - pNext = pPtr->pPg; - lsmFsPageRef(pNext); - while( 1 ){ - Page *pLoad; - u8 *aData; int nData; - - rc = lsmFsDbPageNext(pPtr->pSeg, pNext, 1, &pLoad); - lsmFsPageRelease(pNext); - pNext = pLoad; - if( pNext==0 ) break; - - assert( rc==LSM_OK ); - aData = lsmFsPageData(pNext, &nData); - if( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG)==0 - && pageGetNRec(aData, nData)>0 - ){ - break; - } - } - if( pNext==0 ) break; - segmentPtrSetPage(pPtr, pNext); - - /* This should probably be an LSM_CORRUPT error. */ - assert( rc!=LSM_OK || (pPtr->flags & PGFTR_SKIP_THIS_FLAG) ); - } - - return rc; -} - -static int ptrFwdPointer( - Page *pPage, - int iCell, - Segment *pSeg, - LsmPgno *piPtr, - int *pbFound -){ - Page *pPg = pPage; - int iFirst = iCell; - int rc = LSM_OK; - - do { - Page *pNext = 0; - u8 *aData; - int nData; - - aData = lsmFsPageData(pPg, &nData); - if( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG)==0 ){ - int i; - int nCell = pageGetNRec(aData, nData); - for(i=iFirst; i<nCell; i++){ - u8 eType = *pageGetCell(aData, nData, i); - if( (eType & LSM_START_DELETE)==0 ){ - *pbFound = 1; - *piPtr = pageGetRecordPtr(aData, nData, i) + pageGetPtr(aData, nData); - lsmFsPageRelease(pPg); - return LSM_OK; - } - } - } - - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - iFirst = 0; - }while( pPg && rc==LSM_OK ); - lsmFsPageRelease(pPg); - - *pbFound = 0; - return rc; -} - -static int sortedRhsFirst(MultiCursor *pCsr, Level *pLvl, SegmentPtr *pPtr){ - int rc; - rc = segmentPtrEnd(pCsr, pPtr, 0); - while( pPtr->pPg && rc==LSM_OK ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey - ); - if( res<=0 ) break; - rc = segmentPtrAdvance(pCsr, pPtr, 0); - } - return rc; -} - - -/* -** This function is called as part of a SEEK_GE op on a multi-cursor if the -** FC pointer read from segment *pPtr comes from an entry with the -** LSM_START_DELETE flag set. In this case the pointer value cannot be -** trusted. Instead, the pointer that should be followed is that associated -** with the next entry in *pPtr that does not have LSM_START_DELETE set. -** -** Why the pointers can't be trusted: -** -** -** -** TODO: This is a stop-gap solution: -** -** At the moment, this function is called from within segmentPtrSeek(), -** as part of the initial lsmMCursorSeek() call. However, consider a -** database where the following has occurred: -** -** 1. A range delete removes keys 1..9999 using a range delete. -** 2. Keys 1 through 9999 are reinserted. -** 3. The levels containing the ops in 1. and 2. above are merged. Call -** this level N. Level N contains FC pointers to level N+1. -** -** Then, if the user attempts to query for (key>=2 LIMIT 10), the -** lsmMCursorSeek() call will iterate through 9998 entries searching for a -** pointer down to the level N+1 that is never actually used. It would be -** much better if the multi-cursor could do this lazily - only seek to the -** level (N+1) page after the user has moved the cursor on level N passed -** the big range-delete. -*/ -static int segmentPtrFwdPointer( - MultiCursor *pCsr, /* Multi-cursor pPtr belongs to */ - SegmentPtr *pPtr, /* Segment-pointer to extract FC ptr from */ - LsmPgno *piPtr /* OUT: FC pointer value */ -){ - Level *pLvl = pPtr->pLevel; - Level *pNext = pLvl->pNext; - Page *pPg = pPtr->pPg; - int rc; - int bFound; - LsmPgno iOut = 0; - - if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){ - if( pNext==0 - || (pNext->nRight==0 && pNext->lhs.iRoot) - || (pNext->nRight!=0 && pNext->aRhs[0].iRoot) - ){ - /* Do nothing. The pointer will not be used anyway. */ - return LSM_OK; - } - }else{ - if( pPtr[1].pSeg->iRoot ){ - return LSM_OK; - } - } - - /* Search for a pointer within the current segment. */ - lsmFsPageRef(pPg); - rc = ptrFwdPointer(pPg, pPtr->iCell, pPtr->pSeg, &iOut, &bFound); - - if( rc==LSM_OK && bFound==0 ){ - /* This case happens when pPtr points to the left-hand-side of a segment - ** currently undergoing an incremental merge. In this case, jump to the - ** oldest segment in the right-hand-side of the same level and continue - ** searching. But - do not consider any keys smaller than the levels - ** split-key. */ - SegmentPtr ptr; - - if( pPtr->pLevel->nRight==0 || pPtr->pSeg!=&pPtr->pLevel->lhs ){ - return LSM_CORRUPT_BKPT; - } - - memset(&ptr, 0, sizeof(SegmentPtr)); - ptr.pLevel = pPtr->pLevel; - ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1]; - rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr); - if( rc==LSM_OK ){ - rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound); - ptr.pPg = 0; - } - segmentPtrReset(&ptr, 0); - } - - *piPtr = iOut; - return rc; -} - -static int segmentPtrSeek( - MultiCursor *pCsr, /* Cursor context */ - SegmentPtr *pPtr, /* Pointer to seek */ - int iTopic, /* Key topic to seek to */ - void *pKey, int nKey, /* Key to seek to */ - int eSeek, /* Search bias - see above */ - LsmPgno *piPtr, /* OUT: FC pointer */ - int *pbStop -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int res = 0; /* Result of comparison operation */ - int rc = LSM_OK; - int iMin; - int iMax; - LsmPgno iPtrOut = 0; - - /* If the current page contains an oversized entry, then there are no - ** pointers to one or more of the subsequent pages in the sorted run. - ** The following call ensures that the segment-ptr points to the correct - ** page in this case. */ - rc = segmentPtrSearchOversized(pCsr, pPtr, iTopic, pKey, nKey); - iPtrOut = pPtr->iPtr; - - /* Assert that this page is the right page of this segment for the key - ** that we are searching for. Do this by loading page (iPg-1) and testing - ** that pKey/nKey is greater than all keys on that page, and then by - ** loading (iPg+1) and testing that pKey/nKey is smaller than all - ** the keys it houses. - ** - ** TODO: With range-deletes in the tree, the test described above may fail. - */ -#if 0 - assert( assertKeyLocation(pCsr, pPtr, pKey, nKey) ); -#endif - - assert( pPtr->nCell>0 - || pPtr->pSeg->nSize==1 - || lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg) - ); - if( pPtr->nCell==0 ){ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - }else{ - iMin = 0; - iMax = pPtr->nCell-1; - - while( 1 ){ - int iTry = (iMin+iMax)/2; - void *pKeyT; int nKeyT; /* Key for cell iTry */ - int iTopicT; - - assert( iTry<iMax || iMin==iMax ); - - rc = segmentPtrLoadCell(pPtr, iTry); - if( rc!=LSM_OK ) break; - - segmentPtrKey(pPtr, &pKeyT, &nKeyT); - iTopicT = rtTopic(pPtr->eType); - - res = sortedKeyCompare(xCmp, iTopicT, pKeyT, nKeyT, iTopic, pKey, nKey); - if( res<=0 ){ - iPtrOut = pPtr->iPtr + pPtr->iPgPtr; - } - - if( res==0 || iMin==iMax ){ - break; - }else if( res>0 ){ - iMax = LSM_MAX(iTry-1, iMin); - }else{ - iMin = iTry+1; - } - } - - if( rc==LSM_OK ){ - assert( res==0 || (iMin==iMax && iMin>=0 && iMin<pPtr->nCell) ); - if( res ){ - rc = segmentPtrLoadCell(pPtr, iMin); - } - assert( rc!=LSM_OK || res>0 || iPtrOut==(pPtr->iPtr + pPtr->iPgPtr) ); - - if( rc==LSM_OK ){ - switch( eSeek ){ - case LSM_SEEK_EQ: { - int eType = pPtr->eType; - if( (res<0 && (eType & LSM_START_DELETE)) - || (res>0 && (eType & LSM_END_DELETE)) - || (res==0 && (eType & LSM_POINT_DELETE)) - ){ - *pbStop = 1; - }else if( res==0 && (eType & LSM_INSERT) ){ - lsm_env *pEnv = pCsr->pDb->pEnv; - *pbStop = 1; - pCsr->eType = pPtr->eType; - rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey); - if( rc==LSM_OK ){ - rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal); - } - pCsr->flags |= CURSOR_SEEK_EQ; - } - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - break; - } - case LSM_SEEK_LE: - if( res>0 ) rc = segmentPtrAdvance(pCsr, pPtr, 1); - break; - case LSM_SEEK_GE: { - /* Figure out if we need to 'skip' the pointer forward or not */ - if( (res<=0 && (pPtr->eType & LSM_START_DELETE)) - || (res>0 && (pPtr->eType & LSM_END_DELETE)) - ){ - rc = segmentPtrFwdPointer(pCsr, pPtr, &iPtrOut); - } - if( res<0 && rc==LSM_OK ){ - rc = segmentPtrAdvance(pCsr, pPtr, 0); - } - break; - } - } - } - } - - /* If the cursor seek has found a separator key, and this cursor is - ** supposed to ignore separators keys, advance to the next entry. */ - if( rc==LSM_OK && pPtr->pPg - && segmentPtrIgnoreSeparators(pCsr, pPtr) - && rtIsSeparator(pPtr->eType) - ){ - assert( eSeek!=LSM_SEEK_EQ ); - rc = segmentPtrAdvance(pCsr, pPtr, eSeek==LSM_SEEK_LE); - } - } - - assert( rc!=LSM_OK || assertSeekResult(pCsr,pPtr,iTopic,pKey,nKey,eSeek) ); - *piPtr = iPtrOut; - return rc; -} - -static int seekInBtree( - MultiCursor *pCsr, /* Multi-cursor object */ - Segment *pSeg, /* Seek within this segment */ - int iTopic, - void *pKey, int nKey, /* Key to seek to */ - LsmPgno *aPg, /* OUT: Page numbers */ - Page **ppPg /* OUT: Leaf (sorted-run) page reference */ -){ - int i = 0; - int rc; - LsmPgno iPg; - Page *pPg = 0; - LsmBlob blob = {0, 0, 0}; - - iPg = pSeg->iRoot; - do { - LsmPgno *piFirst = 0; - if( aPg ){ - aPg[i++] = iPg; - piFirst = &aPg[i]; - } - - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, iPg, &pPg); - assert( rc==LSM_OK || pPg==0 ); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer containing page data */ - int nData; /* Size of aData[] in bytes */ - int iMin; - int iMax; - int nRec; - int flags; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( (flags & SEGMENT_BTREE_FLAG)==0 ) break; - - iPg = pageGetPtr(aData, nData); - nRec = pageGetNRec(aData, nData); - - iMin = 0; - iMax = nRec-1; - while( iMax>=iMin ){ - int iTry = (iMin+iMax)/2; - void *pKeyT; int nKeyT; /* Key for cell iTry */ - int iTopicT; /* Topic for key pKeyT/nKeyT */ - LsmPgno iPtr; /* Pointer associated with cell iTry */ - int res; /* (pKey - pKeyT) */ - - rc = pageGetBtreeKey( - pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob - ); - if( rc!=LSM_OK ) break; - if( piFirst && pKeyT==blob.pData ){ - *piFirst = pageGetBtreeRef(pPg, iTry); - piFirst = 0; - i++; - } - - res = sortedKeyCompare( - pCsr->pDb->xCmp, iTopic, pKey, nKey, iTopicT, pKeyT, nKeyT - ); - if( res<0 ){ - iPg = iPtr; - iMax = iTry-1; - }else{ - iMin = iTry+1; - } - } - lsmFsPageRelease(pPg); - pPg = 0; - } - }while( rc==LSM_OK ); - - sortedBlobFree(&blob); - assert( (rc==LSM_OK)==(pPg!=0) ); - if( ppPg ){ - *ppPg = pPg; - }else{ - lsmFsPageRelease(pPg); - } - return rc; -} - -static int seekInSegment( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int iTopic, - void *pKey, int nKey, - LsmPgno iPg, /* Page to search */ - int eSeek, /* Search bias - see above */ - LsmPgno *piPtr, /* OUT: FC pointer */ - int *pbStop /* OUT: Stop search flag */ -){ - LsmPgno iPtr = iPg; - int rc = LSM_OK; - - if( pPtr->pSeg->iRoot ){ - Page *pPg; - assert( pPtr->pSeg->iRoot!=0 ); - rc = seekInBtree(pCsr, pPtr->pSeg, iTopic, pKey, nKey, 0, &pPg); - if( rc==LSM_OK ) segmentPtrSetPage(pPtr, pPg); - }else{ - if( iPtr==0 ){ - iPtr = pPtr->pSeg->iFirst; - } - if( rc==LSM_OK ){ - rc = segmentPtrLoadPage(pCsr->pDb->pFS, pPtr, iPtr); - } - } - - if( rc==LSM_OK ){ - rc = segmentPtrSeek(pCsr, pPtr, iTopic, pKey, nKey, eSeek, piPtr, pbStop); - } - return rc; -} - -/* -** Seek each segment pointer in the array of (pLvl->nRight+1) at aPtr[]. -** -** pbStop: -** This parameter is only significant if parameter eSeek is set to -** LSM_SEEK_EQ. In this case, it is set to true before returning if -** the seek operation is finished. This can happen in two ways: -** -** a) A key matching (pKey/nKey) is found, or -** b) A point-delete or range-delete deleting the key is found. -** -** In case (a), the multi-cursor CURSOR_SEEK_EQ flag is set and the pCsr->key -** and pCsr->val blobs populated before returning. -*/ -static int seekInLevel( - MultiCursor *pCsr, /* Sorted cursor object to seek */ - SegmentPtr *aPtr, /* Pointer to array of (nRhs+1) SPs */ - int eSeek, /* Search bias - see above */ - int iTopic, /* Key topic to search for */ - void *pKey, int nKey, /* Key to search for */ - LsmPgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */ - int *pbStop /* OUT: See above */ -){ - Level *pLvl = aPtr[0].pLevel; /* Level to seek within */ - int rc = LSM_OK; /* Return code */ - LsmPgno iOut = 0; /* Pointer to return to caller */ - int res = -1; /* Result of xCmp(pKey, split) */ - int nRhs = pLvl->nRight; /* Number of right-hand-side segments */ - int bStop = 0; - - /* If this is a composite level (one currently undergoing an incremental - ** merge), figure out if the search key is larger or smaller than the - ** levels split-key. */ - if( nRhs ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - } - - /* If (res<0), then key pKey/nKey is smaller than the split-key (or this - ** is not a composite level and there is no split-key). Search the - ** left-hand-side of the level in this case. */ - if( res<0 ){ - int i; - LsmPgno iPtr = 0; - if( nRhs==0 ) iPtr = *piPgno; - - rc = seekInSegment( - pCsr, &aPtr[0], iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop - ); - if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){ - res = 0; - } - for(i=1; i<=nRhs; i++){ - segmentPtrReset(&aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - - if( res>=0 ){ - int bHit = 0; /* True if at least one rhs is not EOF */ - LsmPgno iPtr = *piPgno; - int i; - segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD); - for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){ - SegmentPtr *pPtr = &aPtr[i]; - iOut = 0; - rc = seekInSegment( - pCsr, pPtr, iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop - ); - iPtr = iOut; - - /* If the segment-pointer has settled on a key that is smaller than - ** the splitkey, invalidate the segment-pointer. */ - if( pPtr->pPg ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ){ - if( pPtr->eType & LSM_START_DELETE ){ - pPtr->eType &= ~LSM_INSERT; - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - pPtr->pVal = 0; - pPtr->nVal = 0; - }else{ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - } - - if( aPtr[i].pKey ) bHit = 1; - } - - if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){ - rc = segmentPtrEnd(pCsr, &aPtr[0], 1); - } - } - - assert( eSeek==LSM_SEEK_EQ || bStop==0 ); - *piPgno = iOut; - *pbStop = bStop; - return rc; -} - -static void multiCursorGetKey( - MultiCursor *pCsr, - int iKey, - int *peType, /* OUT: Key type (SORTED_WRITE etc.) */ - void **ppKey, /* OUT: Pointer to buffer containing key */ - int *pnKey /* OUT: Size of *ppKey in bytes */ -){ - int nKey = 0; - void *pKey = 0; - int eType = 0; - - switch( iKey ){ - case CURSOR_DATA_TREE0: - case CURSOR_DATA_TREE1: { - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - if( lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorKey(pTreeCsr, &eType, &pKey, &nKey); - } - break; - } - - case CURSOR_DATA_SYSTEM: { - Snapshot *pWorker = pCsr->pDb->pWorker; - if( pWorker && (pCsr->flags & CURSOR_FLUSH_FREELIST) ){ - int nEntry = pWorker->freelist.nEntry; - if( pCsr->iFree < (nEntry*2) ){ - FreelistEntry *aEntry = pWorker->freelist.aEntry; - int i = nEntry - 1 - (pCsr->iFree / 2); - u32 iKey2 = 0; - - if( (pCsr->iFree % 2) ){ - eType = LSM_END_DELETE|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk-1; - }else if( aEntry[i].iId>=0 ){ - eType = LSM_INSERT|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk; - - /* If the in-memory entry immediately before this one was a - ** DELETE, and the block number is one greater than the current - ** block number, mark this entry as an "end-delete-range". */ - if( i<(nEntry-1) && aEntry[i+1].iBlk==iKey2+1 && aEntry[i+1].iId<0 ){ - eType |= LSM_END_DELETE; - } - - }else{ - eType = LSM_START_DELETE|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk + 1; - } - - /* If the in-memory entry immediately after this one is a - ** DELETE, and the block number is one less than the current - ** key, mark this entry as an "start-delete-range". */ - if( i>0 && aEntry[i-1].iBlk==iKey2-1 && aEntry[i-1].iId<0 ){ - eType |= LSM_START_DELETE; - } - - pKey = pCsr->pSystemVal; - nKey = 4; - lsmPutU32(pKey, ~iKey2); - } - } - break; - } - - default: { - int iPtr = iKey - CURSOR_DATA_SEGMENT; - assert( iPtr>=0 ); - if( iPtr==pCsr->nPtr ){ - if( pCsr->pBtCsr ){ - pKey = pCsr->pBtCsr->pKey; - nKey = pCsr->pBtCsr->nKey; - eType = pCsr->pBtCsr->eType; - } - }else if( iPtr<pCsr->nPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - if( pPtr->pPg ){ - pKey = pPtr->pKey; - nKey = pPtr->nKey; - eType = pPtr->eType; - } - } - break; - } - } - - if( peType ) *peType = eType; - if( pnKey ) *pnKey = nKey; - if( ppKey ) *ppKey = pKey; -} - -static int sortedDbKeyCompare( - MultiCursor *pCsr, - int iLhsFlags, void *pLhsKey, int nLhsKey, - int iRhsFlags, void *pRhsKey, int nRhsKey -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int res; - - /* Compare the keys, including the system flag. */ - res = sortedKeyCompare(xCmp, - rtTopic(iLhsFlags), pLhsKey, nLhsKey, - rtTopic(iRhsFlags), pRhsKey, nRhsKey - ); - - /* If a key has the LSM_START_DELETE flag set, but not the LSM_INSERT or - ** LSM_POINT_DELETE flags, it is considered a delta larger. This prevents - ** the beginning of an open-ended set from masking a database entry or - ** delete at a lower level. */ - if( res==0 && (pCsr->flags & CURSOR_IGNORE_DELETE) ){ - const int m = LSM_POINT_DELETE|LSM_INSERT|LSM_END_DELETE |LSM_START_DELETE; - int iDel1 = 0; - int iDel2 = 0; - - if( LSM_START_DELETE==(iLhsFlags & m) ) iDel1 = +1; - if( LSM_END_DELETE ==(iLhsFlags & m) ) iDel1 = -1; - if( LSM_START_DELETE==(iRhsFlags & m) ) iDel2 = +1; - if( LSM_END_DELETE ==(iRhsFlags & m) ) iDel2 = -1; - - res = (iDel1 - iDel2); - } - - return res; -} - -static void multiCursorDoCompare(MultiCursor *pCsr, int iOut, int bReverse){ - int i1; - int i2; - int iRes; - void *pKey1; int nKey1; int eType1; - void *pKey2; int nKey2; int eType2; - const int mul = (bReverse ? -1 : 1); - - assert( pCsr->aTree && iOut<pCsr->nTree ); - if( iOut>=(pCsr->nTree/2) ){ - i1 = (iOut - pCsr->nTree/2) * 2; - i2 = i1 + 1; - }else{ - i1 = pCsr->aTree[iOut*2]; - i2 = pCsr->aTree[iOut*2+1]; - } - - multiCursorGetKey(pCsr, i1, &eType1, &pKey1, &nKey1); - multiCursorGetKey(pCsr, i2, &eType2, &pKey2, &nKey2); - - if( pKey1==0 ){ - iRes = i2; - }else if( pKey2==0 ){ - iRes = i1; - }else{ - int res; - - /* Compare the keys */ - res = sortedDbKeyCompare(pCsr, - eType1, pKey1, nKey1, eType2, pKey2, nKey2 - ); - - res = res * mul; - if( res==0 ){ - /* The two keys are identical. Normally, this means that the key from - ** the newer run clobbers the old. However, if the newer key is a - ** separator key, or a range-delete-boundary only, do not allow it - ** to clobber an older entry. */ - int nc1 = (eType1 & (LSM_INSERT|LSM_POINT_DELETE))==0; - int nc2 = (eType2 & (LSM_INSERT|LSM_POINT_DELETE))==0; - iRes = (nc1 > nc2) ? i2 : i1; - }else if( res<0 ){ - iRes = i1; - }else{ - iRes = i2; - } - } - - pCsr->aTree[iOut] = iRes; -} - -/* -** This function advances segment pointer iPtr belonging to multi-cursor -** pCsr forward (bReverse==0) or backward (bReverse!=0). -** -** If the segment pointer points to a segment that is part of a composite -** level, then the following special case is handled. -** -** * If iPtr is the lhs of a composite level, and the cursor is being -** advanced forwards, and segment iPtr is at EOF, move all pointers -** that correspond to rhs segments of the same level to the first -** key in their respective data. -*/ -static int segmentCursorAdvance( - MultiCursor *pCsr, - int iPtr, - int bReverse -){ - int rc; - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - Level *pLvl = pPtr->pLevel; - int bComposite; /* True if pPtr is part of composite level */ - - /* Advance the segment-pointer object. */ - rc = segmentPtrAdvance(pCsr, pPtr, bReverse); - if( rc!=LSM_OK ) return rc; - - bComposite = (pLvl->nRight>0 && pCsr->nPtr>pLvl->nRight); - if( bComposite && pPtr->pPg==0 ){ - int bFix = 0; - if( (bReverse==0)==(pPtr->pSeg==&pLvl->lhs) ){ - int i; - if( bReverse ){ - SegmentPtr *pLhs = &pCsr->aPtr[iPtr - 1 - (pPtr->pSeg - pLvl->aRhs)]; - for(i=0; i<pLvl->nRight; i++){ - if( pLhs[i+1].pPg ) break; - } - if( i==pLvl->nRight ){ - bFix = 1; - rc = segmentPtrEnd(pCsr, pLhs, 1); - } - }else{ - bFix = 1; - for(i=0; rc==LSM_OK && i<pLvl->nRight; i++){ - rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); - } - } - } - - if( bFix ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bReverse); - } - } - } - -#if 0 - if( bComposite && pPtr->pSeg==&pLvl->lhs /* lhs of composite level */ - && bReverse==0 /* csr advanced forwards */ - && pPtr->pPg==0 /* segment at EOF */ - ){ - int i; - for(i=0; rc==LSM_OK && i<pLvl->nRight; i++){ - rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); - } - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, 0); - } - } -#endif - - return rc; -} - -static void mcursorFreeComponents(MultiCursor *pCsr){ - int i; - lsm_env *pEnv = pCsr->pDb->pEnv; - - /* Close the tree cursor, if any. */ - lsmTreeCursorDestroy(pCsr->apTreeCsr[0]); - lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); - - /* Reset the segment pointers */ - for(i=0; i<pCsr->nPtr; i++){ - segmentPtrReset(&pCsr->aPtr[i], 0); - } - - /* And the b-tree cursor, if any */ - btreeCursorFree(pCsr->pBtCsr); - - /* Free allocations */ - lsmFree(pEnv, pCsr->aPtr); - lsmFree(pEnv, pCsr->aTree); - lsmFree(pEnv, pCsr->pSystemVal); - - /* Zero fields */ - pCsr->nPtr = 0; - pCsr->aPtr = 0; - pCsr->nTree = 0; - pCsr->aTree = 0; - pCsr->pSystemVal = 0; - pCsr->apTreeCsr[0] = 0; - pCsr->apTreeCsr[1] = 0; - pCsr->pBtCsr = 0; -} - -void lsmMCursorFreeCache(lsm_db *pDb){ - MultiCursor *p; - MultiCursor *pNext; - for(p=pDb->pCsrCache; p; p=pNext){ - pNext = p->pNext; - lsmMCursorClose(p, 0); - } - pDb->pCsrCache = 0; -} - -/* -** Close the cursor passed as the first argument. -** -** If the bCache parameter is true, then shift the cursor to the pCsrCache -** list for possible reuse instead of actually deleting it. -*/ -void lsmMCursorClose(MultiCursor *pCsr, int bCache){ - if( pCsr ){ - lsm_db *pDb = pCsr->pDb; - MultiCursor **pp; /* Iterator variable */ - - /* The cursor may or may not be currently part of the linked list - ** starting at lsm_db.pCsr. If it is, extract it. */ - for(pp=&pDb->pCsr; *pp; pp=&((*pp)->pNext)){ - if( *pp==pCsr ){ - *pp = pCsr->pNext; - break; - } - } - - if( bCache ){ - int i; /* Used to iterate through segment-pointers */ - - /* Release any page references held by this cursor. */ - assert( !pCsr->pBtCsr ); - for(i=0; i<pCsr->nPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - lsmFsPageRelease(pPtr->pPg); - pPtr->pPg = 0; - } - - /* Reset the tree cursors */ - lsmTreeCursorReset(pCsr->apTreeCsr[0]); - lsmTreeCursorReset(pCsr->apTreeCsr[1]); - - /* Add the cursor to the pCsrCache list */ - pCsr->pNext = pDb->pCsrCache; - pDb->pCsrCache = pCsr; - }else{ - /* Free the allocation used to cache the current key, if any. */ - sortedBlobFree(&pCsr->key); - sortedBlobFree(&pCsr->val); - - /* Free the component cursors */ - mcursorFreeComponents(pCsr); - - /* Free the cursor structure itself */ - lsmFree(pDb->pEnv, pCsr); - } - } -} - -#define TREE_NONE 0 -#define TREE_OLD 1 -#define TREE_BOTH 2 - -/* -** Parameter eTree is one of TREE_OLD or TREE_BOTH. -*/ -static int multiCursorAddTree(MultiCursor *pCsr, Snapshot *pSnap, int eTree){ - int rc = LSM_OK; - lsm_db *db = pCsr->pDb; - - /* Add a tree cursor on the 'old' tree, if it exists. */ - if( eTree!=TREE_NONE - && lsmTreeHasOld(db) - && db->treehdr.iOldLog!=pSnap->iLogOff - ){ - rc = lsmTreeCursorNew(db, 1, &pCsr->apTreeCsr[1]); - } - - /* Add a tree cursor on the 'current' tree, if required. */ - if( rc==LSM_OK && eTree==TREE_BOTH ){ - rc = lsmTreeCursorNew(db, 0, &pCsr->apTreeCsr[0]); - } - - return rc; -} - -static int multiCursorAddRhs(MultiCursor *pCsr, Level *pLvl){ - int i; - int nRhs = pLvl->nRight; - - assert( pLvl->nRight>0 ); - assert( pCsr->aPtr==0 ); - pCsr->aPtr = lsmMallocZero(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nRhs); - if( !pCsr->aPtr ) return LSM_NOMEM_BKPT; - pCsr->nPtr = nRhs; - - for(i=0; i<nRhs; i++){ - pCsr->aPtr[i].pSeg = &pLvl->aRhs[i]; - pCsr->aPtr[i].pLevel = pLvl; - } - - return LSM_OK; -} - -static void multiCursorAddOne(MultiCursor *pCsr, Level *pLvl, int *pRc){ - if( *pRc==LSM_OK ){ - int iPtr = pCsr->nPtr; - int i; - pCsr->aPtr[iPtr].pLevel = pLvl; - pCsr->aPtr[iPtr].pSeg = &pLvl->lhs; - iPtr++; - for(i=0; i<pLvl->nRight; i++){ - pCsr->aPtr[iPtr].pLevel = pLvl; - pCsr->aPtr[iPtr].pSeg = &pLvl->aRhs[i]; - iPtr++; - } - - if( pLvl->nRight && pLvl->pSplitKey==0 ){ - sortedSplitkey(pCsr->pDb, pLvl, pRc); - } - pCsr->nPtr = iPtr; - } -} - -static int multiCursorAddAll(MultiCursor *pCsr, Snapshot *pSnap){ - Level *pLvl; - int nPtr = 0; - int rc = LSM_OK; - - for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ - /* If the LEVEL_INCOMPLETE flag is set, then this function is being - ** called (indirectly) from within a sortedNewToplevel() call to - ** construct pLvl. In this case ignore pLvl - this cursor is going to - ** be used to retrieve a freelist entry from the LSM, and the partially - ** complete level may confuse it. */ - if( pLvl->flags & LEVEL_INCOMPLETE ) continue; - nPtr += (1 + pLvl->nRight); - } - - assert( pCsr->aPtr==0 ); - pCsr->aPtr = lsmMallocZeroRc(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nPtr, &rc); - - for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ - if( (pLvl->flags & LEVEL_INCOMPLETE)==0 ){ - multiCursorAddOne(pCsr, pLvl, &rc); - } - } - - return rc; -} - -static int multiCursorInit(MultiCursor *pCsr, Snapshot *pSnap){ - int rc; - rc = multiCursorAddAll(pCsr, pSnap); - if( rc==LSM_OK ){ - rc = multiCursorAddTree(pCsr, pSnap, TREE_BOTH); - } - pCsr->flags |= (CURSOR_IGNORE_SYSTEM | CURSOR_IGNORE_DELETE); - return rc; -} - -static MultiCursor *multiCursorNew(lsm_db *db, int *pRc){ - MultiCursor *pCsr; - pCsr = (MultiCursor *)lsmMallocZeroRc(db->pEnv, sizeof(MultiCursor), pRc); - if( pCsr ){ - pCsr->pNext = db->pCsr; - db->pCsr = pCsr; - pCsr->pDb = db; - } - return pCsr; -} - - -void lsmSortedRemap(lsm_db *pDb){ - MultiCursor *pCsr; - for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ - int iPtr; - if( pCsr->pBtCsr ){ - btreeCursorLoadKey(pCsr->pBtCsr); - } - for(iPtr=0; iPtr<pCsr->nPtr; iPtr++){ - segmentPtrLoadCell(&pCsr->aPtr[iPtr], pCsr->aPtr[iPtr].iCell); - } - } -} - -static void multiCursorReadSeparators(MultiCursor *pCsr){ - if( pCsr->nPtr>0 ){ - pCsr->flags |= CURSOR_READ_SEPARATORS; - } -} - -/* -** Have this cursor skip over SORTED_DELETE entries. -*/ -static void multiCursorIgnoreDelete(MultiCursor *pCsr){ - if( pCsr ) pCsr->flags |= CURSOR_IGNORE_DELETE; -} - -/* -** If the free-block list is not empty, then have this cursor visit a key -** with (a) the system bit set, and (b) the key "FREELIST" and (c) a value -** blob containing the serialized free-block list. -*/ -static int multiCursorVisitFreelist(MultiCursor *pCsr){ - int rc = LSM_OK; - pCsr->flags |= CURSOR_FLUSH_FREELIST; - pCsr->pSystemVal = lsmMallocRc(pCsr->pDb->pEnv, 4 + 8, &rc); - return rc; -} - -/* -** Allocate and return a new database cursor. -** -** This method should only be called to allocate user cursors. As it may -** recycle a cursor from lsm_db.pCsrCache. -*/ -int lsmMCursorNew( - lsm_db *pDb, /* Database handle */ - MultiCursor **ppCsr /* OUT: Allocated cursor */ -){ - MultiCursor *pCsr = 0; - int rc = LSM_OK; - - if( pDb->pCsrCache ){ - int bOld; /* True if there is an old in-memory tree */ - - /* Remove a cursor from the pCsrCache list and add it to the open list. */ - pCsr = pDb->pCsrCache; - pDb->pCsrCache = pCsr->pNext; - pCsr->pNext = pDb->pCsr; - pDb->pCsr = pCsr; - - /* The cursor can almost be used as is, except that the old in-memory - ** tree cursor may be present and not required, or required and not - ** present. Fix this if required. */ - bOld = (lsmTreeHasOld(pDb) && pDb->treehdr.iOldLog!=pDb->pClient->iLogOff); - if( !bOld && pCsr->apTreeCsr[1] ){ - lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); - pCsr->apTreeCsr[1] = 0; - }else if( bOld && !pCsr->apTreeCsr[1] ){ - rc = lsmTreeCursorNew(pDb, 1, &pCsr->apTreeCsr[1]); - } - - pCsr->flags = (CURSOR_IGNORE_SYSTEM | CURSOR_IGNORE_DELETE); - - }else{ - pCsr = multiCursorNew(pDb, &rc); - if( rc==LSM_OK ) rc = multiCursorInit(pCsr, pDb->pClient); - } - - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - pCsr = 0; - } - assert( (rc==LSM_OK)==(pCsr!=0) ); - *ppCsr = pCsr; - return rc; -} - -static int multiCursorGetVal( - MultiCursor *pCsr, - int iVal, - void **ppVal, - int *pnVal -){ - int rc = LSM_OK; - - *ppVal = 0; - *pnVal = 0; - - switch( iVal ){ - case CURSOR_DATA_TREE0: - case CURSOR_DATA_TREE1: { - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iVal-CURSOR_DATA_TREE0]; - if( lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorValue(pTreeCsr, ppVal, pnVal); - }else{ - *ppVal = 0; - *pnVal = 0; - } - break; - } - - case CURSOR_DATA_SYSTEM: { - Snapshot *pWorker = pCsr->pDb->pWorker; - if( pWorker - && (pCsr->iFree % 2)==0 - && pCsr->iFree < (pWorker->freelist.nEntry*2) - ){ - int iEntry = pWorker->freelist.nEntry - 1 - (pCsr->iFree / 2); - u8 *aVal = &((u8 *)(pCsr->pSystemVal))[4]; - lsmPutU64(aVal, pWorker->freelist.aEntry[iEntry].iId); - *ppVal = aVal; - *pnVal = 8; - } - break; - } - - default: { - int iPtr = iVal-CURSOR_DATA_SEGMENT; - if( iPtr<pCsr->nPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - if( pPtr->pPg ){ - *ppVal = pPtr->pVal; - *pnVal = pPtr->nVal; - } - } - } - } - - assert( rc==LSM_OK || (*ppVal==0 && *pnVal==0) ); - return rc; -} - -static int multiCursorAdvance(MultiCursor *pCsr, int bReverse); - -/* -** This function is called by worker connections to walk the part of the -** free-list stored within the LSM data structure. -*/ -int lsmSortedWalkFreelist( - lsm_db *pDb, /* Database handle */ - int bReverse, /* True to iterate from largest to smallest */ - int (*x)(void *, int, i64), /* Callback function */ - void *pCtx /* First argument to pass to callback */ -){ - MultiCursor *pCsr; /* Cursor used to read db */ - int rc = LSM_OK; /* Return Code */ - Snapshot *pSnap = 0; - - assert( pDb->pWorker ); - if( pDb->bIncrMerge ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->pShmhdr->aSnap1, &pSnap); - if( rc!=LSM_OK ) return rc; - }else{ - pSnap = pDb->pWorker; - } - - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - rc = multiCursorAddAll(pCsr, pSnap); - pCsr->flags |= CURSOR_IGNORE_DELETE; - } - - if( rc==LSM_OK ){ - if( bReverse==0 ){ - rc = lsmMCursorLast(pCsr); - }else{ - rc = lsmMCursorSeek(pCsr, 1, "", 0, LSM_SEEK_GE); - } - - while( rc==LSM_OK && lsmMCursorValid(pCsr) && rtIsSystem(pCsr->eType) ){ - void *pKey; int nKey; - void *pVal = 0; int nVal = 0; - - rc = lsmMCursorKey(pCsr, &pKey, &nKey); - if( rc==LSM_OK ) rc = lsmMCursorValue(pCsr, &pVal, &nVal); - if( rc==LSM_OK && (nKey!=4 || nVal!=8) ) rc = LSM_CORRUPT_BKPT; - - if( rc==LSM_OK ){ - int iBlk; - i64 iSnap; - iBlk = (int)(~(lsmGetU32((u8 *)pKey))); - iSnap = (i64)lsmGetU64((u8 *)pVal); - if( x(pCtx, iBlk, iSnap) ) break; - rc = multiCursorAdvance(pCsr, !bReverse); - } - } - } - - lsmMCursorClose(pCsr, 0); - if( pSnap!=pDb->pWorker ){ - lsmFreeSnapshot(pDb->pEnv, pSnap); - } - - return rc; -} - -int lsmSortedLoadFreelist( - lsm_db *pDb, /* Database handle (must be worker) */ - void **ppVal, /* OUT: Blob containing LSM free-list */ - int *pnVal /* OUT: Size of *ppVal blob in bytes */ -){ - MultiCursor *pCsr; /* Cursor used to retrieve free-list */ - int rc = LSM_OK; /* Return Code */ - - assert( pDb->pWorker ); - assert( *ppVal==0 && *pnVal==0 ); - - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - rc = multiCursorAddAll(pCsr, pDb->pWorker); - pCsr->flags |= CURSOR_IGNORE_DELETE; - } - - if( rc==LSM_OK ){ - rc = lsmMCursorLast(pCsr); - if( rc==LSM_OK - && rtIsWrite(pCsr->eType) && rtIsSystem(pCsr->eType) - && pCsr->key.nData==8 - && 0==memcmp(pCsr->key.pData, "FREELIST", 8) - ){ - void *pVal; int nVal; /* Value read from database */ - rc = lsmMCursorValue(pCsr, &pVal, &nVal); - if( rc==LSM_OK ){ - *ppVal = lsmMallocRc(pDb->pEnv, nVal, &rc); - if( *ppVal ){ - memcpy(*ppVal, pVal, nVal); - *pnVal = nVal; - } - } - } - - lsmMCursorClose(pCsr, 0); - } - - return rc; -} - -static int multiCursorAllocTree(MultiCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->aTree==0 ){ - int nByte; /* Bytes of space to allocate */ - int nMin; /* Total number of cursors being merged */ - - nMin = CURSOR_DATA_SEGMENT + pCsr->nPtr + (pCsr->pBtCsr!=0); - pCsr->nTree = 2; - while( pCsr->nTree<nMin ){ - pCsr->nTree = pCsr->nTree*2; - } - - nByte = sizeof(int)*pCsr->nTree*2; - pCsr->aTree = (int *)lsmMallocZeroRc(pCsr->pDb->pEnv, nByte, &rc); - } - return rc; -} - -static void multiCursorCacheKey(MultiCursor *pCsr, int *pRc){ - if( *pRc==LSM_OK ){ - void *pKey; - int nKey; - multiCursorGetKey(pCsr, pCsr->aTree[1], &pCsr->eType, &pKey, &nKey); - *pRc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->key, pKey, nKey); - } -} - -#ifdef LSM_DEBUG_EXPENSIVE -static void assertCursorTree(MultiCursor *pCsr){ - int bRev = !!(pCsr->flags & CURSOR_PREV_OK); - int *aSave = pCsr->aTree; - int nSave = pCsr->nTree; - int rc; - - pCsr->aTree = 0; - pCsr->nTree = 0; - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - - assert( nSave==pCsr->nTree - && 0==memcmp(aSave, pCsr->aTree, sizeof(int)*nSave) - ); - - lsmFree(pCsr->pDb->pEnv, pCsr->aTree); - } - - pCsr->aTree = aSave; - pCsr->nTree = nSave; -} -#else -# define assertCursorTree(x) -#endif - -static int mcursorLocationOk(MultiCursor *pCsr, int bDeleteOk){ - int eType = pCsr->eType; - int iKey; - int i; - int rdmask; - - assert( pCsr->flags & (CURSOR_NEXT_OK|CURSOR_PREV_OK) ); - assertCursorTree(pCsr); - - rdmask = (pCsr->flags & CURSOR_NEXT_OK) ? LSM_END_DELETE : LSM_START_DELETE; - - /* If the cursor does not currently point to an actual database key (i.e. - ** it points to a delete key, or the start or end of a range-delete), and - ** the CURSOR_IGNORE_DELETE flag is set, skip past this entry. */ - if( (pCsr->flags & CURSOR_IGNORE_DELETE) && bDeleteOk==0 ){ - if( (eType & LSM_INSERT)==0 ) return 0; - } - - /* If the cursor points to a system key (free-list entry), and the - ** CURSOR_IGNORE_SYSTEM flag is set, skip this entry. */ - if( (pCsr->flags & CURSOR_IGNORE_SYSTEM) && rtTopic(eType)!=0 ){ - return 0; - } - -#ifndef NDEBUG - /* This block fires assert() statements to check one of the assumptions - ** in the comment below - that if the lhs sub-cursor of a level undergoing - ** a merge is valid, then all the rhs sub-cursors must be at EOF. - ** - ** Also assert that all rhs sub-cursors are either at EOF or point to - ** a key that is not less than the level split-key. */ - for(i=0; i<pCsr->nPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - Level *pLvl = pPtr->pLevel; - if( pLvl->nRight && pPtr->pPg ){ - if( pPtr->pSeg==&pLvl->lhs ){ - int j; - for(j=0; j<pLvl->nRight; j++) assert( pPtr[j+1].pPg==0 ); - }else{ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - assert( res>=0 ); - } - } - } -#endif - - /* Now check if this key has already been deleted by a range-delete. If - ** so, skip past it. - ** - ** Assume, for the moment, that the tree contains no levels currently - ** undergoing incremental merge, and that this cursor is iterating forwards - ** through the database keys. The cursor currently points to a key in - ** level L. This key has already been deleted if any of the sub-cursors - ** that point to levels newer than L (or to the in-memory tree) point to - ** a key greater than the current key with the LSM_END_DELETE flag set. - ** - ** Or, if the cursor is iterating backwards through data keys, if any - ** such sub-cursor points to a key smaller than the current key with the - ** LSM_START_DELETE flag set. - ** - ** Why it works with levels undergoing a merge too: - ** - ** When a cursor iterates forwards, the sub-cursors for the rhs of a - ** level are only activated once the lhs reaches EOF. So when iterating - ** forwards, the keys visited are the same as if the level was completely - ** merged. - ** - ** If the cursor is iterating backwards, then the lhs sub-cursor is not - ** initialized until the last of the rhs sub-cursors has reached EOF. - ** Additionally, if the START_DELETE flag is set on the last entry (in - ** reverse order - so the entry with the smallest key) of a rhs sub-cursor, - ** then a pseudo-key equal to the levels split-key with the END_DELETE - ** flag set is visited by the sub-cursor. - */ - iKey = pCsr->aTree[1]; - for(i=0; i<iKey; i++){ - int csrflags; - multiCursorGetKey(pCsr, i, &csrflags, 0, 0); - if( (rdmask & csrflags) ){ - const int SD_ED = (LSM_START_DELETE|LSM_END_DELETE); - if( (csrflags & SD_ED)==SD_ED - || (pCsr->flags & CURSOR_IGNORE_DELETE)==0 - ){ - void *pKey; int nKey; - multiCursorGetKey(pCsr, i, 0, &pKey, &nKey); - if( 0==sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(eType), pCsr->key.pData, pCsr->key.nData, - rtTopic(csrflags), pKey, nKey - )){ - continue; - } - } - return 0; - } - } - - /* The current cursor position is one this cursor should visit. Return 1. */ - return 1; -} - -static int multiCursorSetupTree(MultiCursor *pCsr, int bRev){ - int rc; - - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - } - - assertCursorTree(pCsr); - multiCursorCacheKey(pCsr, &rc); - - if( rc==LSM_OK && mcursorLocationOk(pCsr, 0)==0 ){ - rc = multiCursorAdvance(pCsr, bRev); - } - return rc; -} - - -static int multiCursorEnd(MultiCursor *pCsr, int bLast){ - int rc = LSM_OK; - int i; - - pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); - pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK); - pCsr->iFree = 0; - - /* Position the two in-memory tree cursors */ - for(i=0; rc==LSM_OK && i<2; i++){ - if( pCsr->apTreeCsr[i] ){ - rc = lsmTreeCursorEnd(pCsr->apTreeCsr[i], bLast); - } - } - - for(i=0; rc==LSM_OK && i<pCsr->nPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - Level *pLvl = pPtr->pLevel; - int iRhs; - int bHit = 0; - - if( bLast ){ - for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){ - rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); - if( pPtr[iRhs+1].pPg ) bHit = 1; - } - if( bHit==0 && rc==LSM_OK ){ - rc = segmentPtrEnd(pCsr, pPtr, 1); - }else{ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - }else{ - int bLhs = (pPtr->pSeg==&pLvl->lhs); - assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] ); - - if( bLhs ){ - rc = segmentPtrEnd(pCsr, pPtr, 0); - if( pPtr->pKey ) bHit = 1; - } - for(iRhs=0; iRhs<pLvl->nRight && rc==LSM_OK; iRhs++){ - if( bHit ){ - segmentPtrReset(&pPtr[iRhs+1], LSM_SEGMENTPTR_FREE_THRESHOLD); - }else{ - rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]); - } - } - } - i += pLvl->nRight; - } - - /* And the b-tree cursor, if applicable */ - if( rc==LSM_OK && pCsr->pBtCsr ){ - assert( bLast==0 ); - rc = btreeCursorFirst(pCsr->pBtCsr); - } - - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(pCsr, bLast); - } - - return rc; -} - - -int mcursorSave(MultiCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->aTree ){ - int iTree = pCsr->aTree[1]; - if( iTree==CURSOR_DATA_TREE0 || iTree==CURSOR_DATA_TREE1 ){ - multiCursorCacheKey(pCsr, &rc); - } - } - mcursorFreeComponents(pCsr); - return rc; -} - -int mcursorRestore(lsm_db *pDb, MultiCursor *pCsr){ - int rc; - rc = multiCursorInit(pCsr, pDb->pClient); - if( rc==LSM_OK && pCsr->key.pData ){ - rc = lsmMCursorSeek(pCsr, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, +1 - ); - } - return rc; -} - -int lsmSaveCursors(lsm_db *pDb){ - int rc = LSM_OK; - MultiCursor *pCsr; - - for(pCsr=pDb->pCsr; rc==LSM_OK && pCsr; pCsr=pCsr->pNext){ - rc = mcursorSave(pCsr); - } - return rc; -} - -int lsmRestoreCursors(lsm_db *pDb){ - int rc = LSM_OK; - MultiCursor *pCsr; - - for(pCsr=pDb->pCsr; rc==LSM_OK && pCsr; pCsr=pCsr->pNext){ - rc = mcursorRestore(pDb, pCsr); - } - return rc; -} - -int lsmMCursorFirst(MultiCursor *pCsr){ - return multiCursorEnd(pCsr, 0); -} - -int lsmMCursorLast(MultiCursor *pCsr){ - return multiCursorEnd(pCsr, 1); -} - -lsm_db *lsmMCursorDb(MultiCursor *pCsr){ - return pCsr->pDb; -} - -void lsmMCursorReset(MultiCursor *pCsr){ - int i; - lsmTreeCursorReset(pCsr->apTreeCsr[0]); - lsmTreeCursorReset(pCsr->apTreeCsr[1]); - for(i=0; i<pCsr->nPtr; i++){ - segmentPtrReset(&pCsr->aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); - } - pCsr->key.nData = 0; -} - -static int treeCursorSeek( - MultiCursor *pCsr, - TreeCursor *pTreeCsr, - void *pKey, int nKey, - int eSeek, - int *pbStop -){ - int rc = LSM_OK; - if( pTreeCsr ){ - int res = 0; - lsmTreeCursorSeek(pTreeCsr, pKey, nKey, &res); - switch( eSeek ){ - case LSM_SEEK_EQ: { - int eType = lsmTreeCursorFlags(pTreeCsr); - if( (res<0 && (eType & LSM_START_DELETE)) - || (res>0 && (eType & LSM_END_DELETE)) - || (res==0 && (eType & LSM_POINT_DELETE)) - ){ - *pbStop = 1; - }else if( res==0 && (eType & LSM_INSERT) ){ - lsm_env *pEnv = pCsr->pDb->pEnv; - void *p; int n; /* Key/value from tree-cursor */ - *pbStop = 1; - pCsr->flags |= CURSOR_SEEK_EQ; - rc = lsmTreeCursorKey(pTreeCsr, &pCsr->eType, &p, &n); - if( rc==LSM_OK ) rc = sortedBlobSet(pEnv, &pCsr->key, p, n); - if( rc==LSM_OK ) rc = lsmTreeCursorValue(pTreeCsr, &p, &n); - if( rc==LSM_OK ) rc = sortedBlobSet(pEnv, &pCsr->val, p, n); - } - lsmTreeCursorReset(pTreeCsr); - break; - } - case LSM_SEEK_GE: - if( res<0 && lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorNext(pTreeCsr); - } - break; - default: - if( res>0 ){ - assert( lsmTreeCursorValid(pTreeCsr) ); - lsmTreeCursorPrev(pTreeCsr); - } - break; - } - } - return rc; -} - - -/* -** Seek the cursor. -*/ -int lsmMCursorSeek( - MultiCursor *pCsr, - int iTopic, - void *pKey, int nKey, - int eSeek -){ - int eESeek = eSeek; /* Effective eSeek parameter */ - int bStop = 0; /* Set to true to halt search operation */ - int rc = LSM_OK; /* Return code */ - int iPtr = 0; /* Used to iterate through pCsr->aPtr[] */ - LsmPgno iPgno = 0; /* FC pointer value */ - - assert( pCsr->apTreeCsr[0]==0 || iTopic==0 ); - assert( pCsr->apTreeCsr[1]==0 || iTopic==0 ); - - if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE; - - assert( eESeek==LSM_SEEK_EQ || eESeek==LSM_SEEK_LE || eESeek==LSM_SEEK_GE ); - assert( (pCsr->flags & CURSOR_FLUSH_FREELIST)==0 ); - assert( pCsr->nPtr==0 || pCsr->aPtr[0].pLevel ); - - pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); - rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[0], pKey, nKey, eESeek, &bStop); - if( rc==LSM_OK && bStop==0 ){ - rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[1], pKey, nKey, eESeek, &bStop); - } - - /* Seek all segment pointers. */ - for(iPtr=0; iPtr<pCsr->nPtr && rc==LSM_OK && bStop==0; iPtr++){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - assert( pPtr->pSeg==&pPtr->pLevel->lhs ); - rc = seekInLevel(pCsr, pPtr, eESeek, iTopic, pKey, nKey, &iPgno, &bStop); - iPtr += pPtr->pLevel->nRight; - } - - if( eSeek!=LSM_SEEK_EQ ){ - if( rc==LSM_OK ){ - rc = multiCursorAllocTree(pCsr); - } - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, eESeek==LSM_SEEK_LE); - } - if( eSeek==LSM_SEEK_GE ) pCsr->flags |= CURSOR_NEXT_OK; - if( eSeek==LSM_SEEK_LE ) pCsr->flags |= CURSOR_PREV_OK; - } - - multiCursorCacheKey(pCsr, &rc); - if( rc==LSM_OK && eSeek!=LSM_SEEK_LEFAST && 0==mcursorLocationOk(pCsr, 0) ){ - switch( eESeek ){ - case LSM_SEEK_EQ: - lsmMCursorReset(pCsr); - break; - case LSM_SEEK_GE: - rc = lsmMCursorNext(pCsr); - break; - default: - rc = lsmMCursorPrev(pCsr); - break; - } - } - } - - return rc; -} - -int lsmMCursorValid(MultiCursor *pCsr){ - int res = 0; - if( pCsr->flags & CURSOR_SEEK_EQ ){ - res = 1; - }else if( pCsr->aTree ){ - int iKey = pCsr->aTree[1]; - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - res = lsmTreeCursorValid(pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]); - }else{ - void *pKey; - multiCursorGetKey(pCsr, iKey, 0, &pKey, 0); - res = pKey!=0; - } - } - return res; -} - -static int mcursorAdvanceOk( - MultiCursor *pCsr, - int bReverse, - int *pRc -){ - void *pNew; /* Pointer to buffer containing new key */ - int nNew; /* Size of buffer pNew in bytes */ - int eNewType; /* Type of new record */ - - if( *pRc ) return 1; - - /* Check the current key value. If it is not greater than (if bReverse==0) - ** or less than (if bReverse!=0) the key currently cached in pCsr->key, - ** then the cursor has not yet been successfully advanced. - */ - multiCursorGetKey(pCsr, pCsr->aTree[1], &eNewType, &pNew, &nNew); - if( pNew ){ - int typemask = (pCsr->flags & CURSOR_IGNORE_DELETE) ? ~(0) : LSM_SYSTEMKEY; - int res = sortedDbKeyCompare(pCsr, - eNewType & typemask, pNew, nNew, - pCsr->eType & typemask, pCsr->key.pData, pCsr->key.nData - ); - - if( (bReverse==0 && res<=0) || (bReverse!=0 && res>=0) ){ - return 0; - } - - multiCursorCacheKey(pCsr, pRc); - assert( pCsr->eType==eNewType ); - - /* If this cursor is configured to skip deleted keys, and the current - ** cursor points to a SORTED_DELETE entry, then the cursor has not been - ** successfully advanced. - ** - ** Similarly, if the cursor is configured to skip system keys and the - ** current cursor points to a system key, it has not yet been advanced. - */ - if( *pRc==LSM_OK && 0==mcursorLocationOk(pCsr, 0) ) return 0; - } - return 1; -} - -static void flCsrAdvance(MultiCursor *pCsr){ - assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); - if( pCsr->iFree % 2 ){ - pCsr->iFree++; - }else{ - int nEntry = pCsr->pDb->pWorker->freelist.nEntry; - FreelistEntry *aEntry = pCsr->pDb->pWorker->freelist.aEntry; - - int i = nEntry - 1 - (pCsr->iFree / 2); - - /* If the current entry is a delete and the "end-delete" key will not - ** be attached to the next entry, increment iFree by 1 only. */ - if( aEntry[i].iId<0 ){ - while( 1 ){ - if( i==0 || aEntry[i-1].iBlk!=aEntry[i].iBlk-1 ){ - pCsr->iFree--; - break; - } - if( aEntry[i-1].iId>=0 ) break; - pCsr->iFree += 2; - i--; - } - } - pCsr->iFree += 2; - } -} - -static int multiCursorAdvance(MultiCursor *pCsr, int bReverse){ - int rc = LSM_OK; /* Return Code */ - if( lsmMCursorValid(pCsr) ){ - do { - int iKey = pCsr->aTree[1]; - - assertCursorTree(pCsr); - - /* If this multi-cursor is advancing forwards, and the sub-cursor - ** being advanced is the one that separator keys may be being read - ** from, record the current absolute pointer value. */ - if( pCsr->pPrevMergePtr ){ - if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ - assert( pCsr->pBtCsr ); - *pCsr->pPrevMergePtr = pCsr->pBtCsr->iPtr; - }else if( pCsr->pBtCsr==0 && pCsr->nPtr>0 - && iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr-1) - ){ - SegmentPtr *pPtr = &pCsr->aPtr[iKey-CURSOR_DATA_SEGMENT]; - *pCsr->pPrevMergePtr = pPtr->iPtr+pPtr->iPgPtr; - } - } - - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - if( bReverse ){ - rc = lsmTreeCursorPrev(pTreeCsr); - }else{ - rc = lsmTreeCursorNext(pTreeCsr); - } - }else if( iKey==CURSOR_DATA_SYSTEM ){ - assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); - assert( bReverse==0 ); - flCsrAdvance(pCsr); - }else if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ - assert( bReverse==0 && pCsr->pBtCsr ); - rc = btreeCursorNext(pCsr->pBtCsr); - }else{ - rc = segmentCursorAdvance(pCsr, iKey-CURSOR_DATA_SEGMENT, bReverse); - } - if( rc==LSM_OK ){ - int i; - for(i=(iKey+pCsr->nTree)/2; i>0; i=i/2){ - multiCursorDoCompare(pCsr, i, bReverse); - } - assertCursorTree(pCsr); - } - }while( mcursorAdvanceOk(pCsr, bReverse, &rc)==0 ); - } - return rc; -} - -int lsmMCursorNext(MultiCursor *pCsr){ - if( (pCsr->flags & CURSOR_NEXT_OK)==0 ) return LSM_MISUSE_BKPT; - return multiCursorAdvance(pCsr, 0); -} - -int lsmMCursorPrev(MultiCursor *pCsr){ - if( (pCsr->flags & CURSOR_PREV_OK)==0 ) return LSM_MISUSE_BKPT; - return multiCursorAdvance(pCsr, 1); -} - -int lsmMCursorKey(MultiCursor *pCsr, void **ppKey, int *pnKey){ - if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ - *pnKey = pCsr->key.nData; - *ppKey = pCsr->key.pData; - }else{ - int iKey = pCsr->aTree[1]; - - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - lsmTreeCursorKey(pTreeCsr, 0, ppKey, pnKey); - }else{ - int nKey; - -#ifndef NDEBUG - void *pKey; - int eType; - multiCursorGetKey(pCsr, iKey, &eType, &pKey, &nKey); - assert( eType==pCsr->eType ); - assert( nKey==pCsr->key.nData ); - assert( memcmp(pKey, pCsr->key.pData, nKey)==0 ); -#endif - - nKey = pCsr->key.nData; - if( nKey==0 ){ - *ppKey = 0; - }else{ - *ppKey = pCsr->key.pData; - } - *pnKey = nKey; - } - } - return LSM_OK; -} - -/* -** Compare the current key that cursor csr points to with pKey/nKey. Set -** *piRes to the result and return LSM_OK. -*/ -int lsm_csr_cmp(lsm_cursor *csr, const void *pKey, int nKey, int *piRes){ - MultiCursor *pCsr = (MultiCursor *)csr; - void *pCsrkey; int nCsrkey; - int rc; - rc = lsmMCursorKey(pCsr, &pCsrkey, &nCsrkey); - if( rc==LSM_OK ){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - *piRes = sortedKeyCompare(xCmp, 0, pCsrkey, nCsrkey, 0, (void *)pKey, nKey); - } - return rc; -} - -int lsmMCursorValue(MultiCursor *pCsr, void **ppVal, int *pnVal){ - void *pVal; - int nVal; - int rc; - if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ - rc = LSM_OK; - nVal = pCsr->val.nData; - pVal = pCsr->val.pData; - }else{ - - assert( pCsr->aTree ); - assert( mcursorLocationOk(pCsr, (pCsr->flags & CURSOR_IGNORE_DELETE)) ); - - rc = multiCursorGetVal(pCsr, pCsr->aTree[1], &pVal, &nVal); - if( pVal && rc==LSM_OK ){ - rc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->val, pVal, nVal); - pVal = pCsr->val.pData; - } - - if( rc!=LSM_OK ){ - pVal = 0; - nVal = 0; - } - } - *ppVal = pVal; - *pnVal = nVal; - return rc; -} - -int lsmMCursorType(MultiCursor *pCsr, int *peType){ - assert( pCsr->aTree ); - multiCursorGetKey(pCsr, pCsr->aTree[1], peType, 0, 0); - return LSM_OK; -} - -/* -** Buffer aData[], size nData, is assumed to contain a valid b-tree -** hierarchy page image. Return the offset in aData[] of the next free -** byte in the data area (where a new cell may be written if there is -** space). -*/ -static int mergeWorkerPageOffset(u8 *aData, int nData){ - int nRec; - int iOff; - int nKey; - int eType; - i64 nDummy; - - - nRec = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); - iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec-1)]); - eType = aData[iOff++]; - assert( eType==0 - || eType==(LSM_SYSTEMKEY|LSM_SEPARATOR) - || eType==(LSM_SEPARATOR) - ); - - iOff += lsmVarintGet64(&aData[iOff], &nDummy); - iOff += lsmVarintGet32(&aData[iOff], &nKey); - - return iOff + (eType ? nKey : 0); -} - -/* -** Following a checkpoint operation, database pages that are part of the -** checkpointed state of the LSM are deemed read-only. This includes the -** right-most page of the b-tree hierarchy of any separators array under -** construction, and all pages between it and the b-tree root, inclusive. -** This is a problem, as when further pages are appended to the separators -** array, entries must be added to the indicated b-tree hierarchy pages. -** -** This function copies all such b-tree pages to new locations, so that -** they can be modified as required. -** -** The complication is that not all database pages are the same size - due -** to the way the file.c module works some (the first and last in each block) -** are 4 bytes smaller than the others. -*/ -static int mergeWorkerMoveHierarchy( - MergeWorker *pMW, /* Merge worker */ - int bSep /* True for separators run */ -){ - lsm_db *pDb = pMW->pDb; /* Database handle */ - int rc = LSM_OK; /* Return code */ - int i; - Page **apHier = pMW->hier.apHier; - int nHier = pMW->hier.nHier; - - for(i=0; rc==LSM_OK && i<nHier; i++){ - Page *pNew = 0; - rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &pNew); - assert( rc==LSM_OK ); - - if( rc==LSM_OK ){ - u8 *a1; int n1; - u8 *a2; int n2; - - a1 = fsPageData(pNew, &n1); - a2 = fsPageData(apHier[i], &n2); - - assert( n1==n2 || n1+4==n2 ); - - if( n1==n2 ){ - memcpy(a1, a2, n2); - }else{ - int nEntry = pageGetNRec(a2, n2); - int iEof1 = SEGMENT_EOF(n1, nEntry); - int iEof2 = SEGMENT_EOF(n2, nEntry); - - memcpy(a1, a2, iEof2 - 4); - memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); - } - - lsmFsPageRelease(apHier[i]); - apHier[i] = pNew; - -#if 0 - assert( n1==n2 || n1+4==n2 || n2+4==n1 ); - if( n1>=n2 ){ - /* If n1 (size of the new page) is equal to or greater than n2 (the - ** size of the old page), then copy the data into the new page. If - ** n1==n2, this could be done with a single memcpy(). However, - ** since sometimes n1>n2, the page content and footer must be copied - ** separately. */ - int nEntry = pageGetNRec(a2, n2); - int iEof1 = SEGMENT_EOF(n1, nEntry); - int iEof2 = SEGMENT_EOF(n2, nEntry); - memcpy(a1, a2, iEof2); - memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); - lsmFsPageRelease(apHier[i]); - apHier[i] = pNew; - }else{ - lsmPutU16(&a1[SEGMENT_FLAGS_OFFSET(n1)], SEGMENT_BTREE_FLAG); - lsmPutU16(&a1[SEGMENT_NRECORD_OFFSET(n1)], 0); - lsmPutU64(&a1[SEGMENT_POINTER_OFFSET(n1)], 0); - i = i - 1; - lsmFsPageRelease(pNew); - } -#endif - } - } - -#ifdef LSM_DEBUG - if( rc==LSM_OK ){ - for(i=0; i<nHier; i++) assert( lsmFsPageWritable(apHier[i]) ); - } -#endif - - return rc; -} - -/* -** Allocate and populate the MergeWorker.apHier[] array. -*/ -static int mergeWorkerLoadHierarchy(MergeWorker *pMW){ - int rc = LSM_OK; - Segment *pSeg; - Hierarchy *p; - - pSeg = &pMW->pLevel->lhs; - p = &pMW->hier; - - if( p->apHier==0 && pSeg->iRoot!=0 ){ - FileSystem *pFS = pMW->pDb->pFS; - lsm_env *pEnv = pMW->pDb->pEnv; - Page **apHier = 0; - int nHier = 0; - LsmPgno iPg = pSeg->iRoot; - - do { - Page *pPg = 0; - u8 *aData; - int nData; - int flags; - - rc = lsmFsDbPageGet(pFS, pSeg, iPg, &pPg); - if( rc!=LSM_OK ) break; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( flags&SEGMENT_BTREE_FLAG ){ - Page **apNew = (Page **)lsmRealloc( - pEnv, apHier, sizeof(Page *)*(nHier+1) - ); - if( apNew==0 ){ - rc = LSM_NOMEM_BKPT; - break; - } - apHier = apNew; - memmove(&apHier[1], &apHier[0], sizeof(Page *) * nHier); - nHier++; - - apHier[0] = pPg; - iPg = pageGetPtr(aData, nData); - }else{ - lsmFsPageRelease(pPg); - break; - } - }while( 1 ); - - if( rc==LSM_OK ){ - u8 *aData; - int nData; - aData = fsPageData(apHier[0], &nData); - pMW->aSave[0].iPgno = pageGetPtr(aData, nData); - p->nHier = nHier; - p->apHier = apHier; - rc = mergeWorkerMoveHierarchy(pMW, 0); - }else{ - int i; - for(i=0; i<nHier; i++){ - lsmFsPageRelease(apHier[i]); - } - lsmFree(pEnv, apHier); - } - } - - return rc; -} - -/* -** B-tree pages use almost the same format as regular pages. The -** differences are: -** -** 1. The record format is (usually, see below) as follows: -** -** + Type byte (always SORTED_SEPARATOR or SORTED_SYSTEM_SEPARATOR), -** + Absolute pointer value (varint), -** + Number of bytes in key (varint), -** + LsmBlob containing key data. -** -** 2. All pointer values are stored as absolute values (not offsets -** relative to the footer pointer value). -** -** 3. Each pointer that is part of a record points to a page that -** contains keys smaller than the records key (note: not "equal to or -** smaller than - smaller than"). -** -** 4. The pointer in the page footer of a b-tree page points to a page -** that contains keys equal to or larger than the largest key on the -** b-tree page. -** -** The reason for having the page footer pointer point to the right-child -** (instead of the left) is that doing things this way makes the -** mergeWorkerMoveHierarchy() operation less complicated (since the pointers -** that need to be updated are all stored as fixed-size integers within the -** page footer, not varints in page records). -** -** Records may not span b-tree pages. If this function is called to add a -** record larger than (page-size / 4) bytes, then a pointer to the indexed -** array page that contains the main record is added to the b-tree instead. -** In this case the record format is: -** -** + 0x00 byte (1 byte) -** + Absolute pointer value (varint), -** + Absolute page number of page containing key (varint). -** -** See function seekInBtree() for the code that traverses b-tree pages. -*/ - -static int mergeWorkerBtreeWrite( - MergeWorker *pMW, - u8 eType, - LsmPgno iPtr, - LsmPgno iKeyPg, - void *pKey, - int nKey -){ - Hierarchy *p = &pMW->hier; - lsm_db *pDb = pMW->pDb; /* Database handle */ - int rc = LSM_OK; /* Return Code */ - int iLevel; /* Level of b-tree hierarchy to write to */ - int nData; /* Size of aData[] in bytes */ - u8 *aData; /* Page data for level iLevel */ - int iOff; /* Offset on b-tree page to write record to */ - int nRec; /* Initial number of records on b-tree page */ - - /* iKeyPg should be zero for an ordinary b-tree key, or non-zero for an - ** indirect key. The flags byte for an indirect key is 0x00. */ - assert( (eType==0)==(iKeyPg!=0) ); - - /* The MergeWorker.apHier[] array contains the right-most leaf of the b-tree - ** hierarchy, the root node, and all nodes that lie on the path between. - ** apHier[0] is the right-most leaf and apHier[pMW->nHier-1] is the current - ** root page. - ** - ** This loop searches for a node with enough space to store the key on, - ** starting with the leaf and iterating up towards the root. When the loop - ** exits, the key may be written to apHier[iLevel]. */ - for(iLevel=0; iLevel<=p->nHier; iLevel++){ - int nByte; /* Number of free bytes required */ - - if( iLevel==p->nHier ){ - /* Extend the array and allocate a new root page. */ - Page **aNew; - aNew = (Page **)lsmRealloc( - pMW->pDb->pEnv, p->apHier, sizeof(Page *)*(p->nHier+1) - ); - if( !aNew ){ - return LSM_NOMEM_BKPT; - } - p->apHier = aNew; - }else{ - Page *pOld; - int nFree; - - /* If the key will fit on this page, break out of the loop here. - ** The new entry will be written to page apHier[iLevel]. */ - pOld = p->apHier[iLevel]; - assert( lsmFsPageWritable(pOld) ); - aData = fsPageData(pOld, &nData); - if( eType==0 ){ - nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen64(iKeyPg); - }else{ - nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen32(nKey) + nKey; - } - - nRec = pageGetNRec(aData, nData); - nFree = SEGMENT_EOF(nData, nRec) - mergeWorkerPageOffset(aData, nData); - if( nByte<=nFree ) break; - - /* Otherwise, this page is full. Set the right-hand-child pointer - ** to iPtr and release it. */ - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); - assert( lsmFsPageNumber(pOld)==0 ); - rc = lsmFsPagePersist(pOld); - if( rc==LSM_OK ){ - iPtr = lsmFsPageNumber(pOld); - lsmFsPageRelease(pOld); - } - } - - /* Allocate a new page for apHier[iLevel]. */ - p->apHier[iLevel] = 0; - if( rc==LSM_OK ){ - rc = lsmFsSortedAppend( - pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &p->apHier[iLevel] - ); - } - if( rc!=LSM_OK ) return rc; - - aData = fsPageData(p->apHier[iLevel], &nData); - memset(aData, 0, nData); - lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], SEGMENT_BTREE_FLAG); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], 0); - - if( iLevel==p->nHier ){ - p->nHier++; - break; - } - } - - /* Write the key into page apHier[iLevel]. */ - aData = fsPageData(p->apHier[iLevel], &nData); - iOff = mergeWorkerPageOffset(aData, nData); - nRec = pageGetNRec(aData, nData); - lsmPutU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec)], (u16)iOff); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1)); - if( eType==0 ){ - aData[iOff++] = 0x00; - iOff += lsmVarintPut64(&aData[iOff], iPtr); - iOff += lsmVarintPut64(&aData[iOff], iKeyPg); - }else{ - aData[iOff++] = eType; - iOff += lsmVarintPut64(&aData[iOff], iPtr); - iOff += lsmVarintPut32(&aData[iOff], nKey); - memcpy(&aData[iOff], pKey, nKey); - } - - return rc; -} - -static int mergeWorkerBtreeIndirect(MergeWorker *pMW){ - int rc = LSM_OK; - if( pMW->iIndirect ){ - LsmPgno iKeyPg = pMW->aSave[1].iPgno; - rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0); - pMW->iIndirect = 0; - } - return rc; -} - -/* -** Append the database key (iTopic/pKey/nKey) to the b-tree under -** construction. This key has not yet been written to a segment page. -** The pointer that will accompany the new key in the b-tree - that -** points to the completed segment page that contains keys smaller than -** (pKey/nKey) is currently stored in pMW->aSave[0].iPgno. -*/ -static int mergeWorkerPushHierarchy( - MergeWorker *pMW, /* Merge worker object */ - int iTopic, /* Topic value for this key */ - void *pKey, /* Pointer to key buffer */ - int nKey /* Size of pKey buffer in bytes */ -){ - int rc = LSM_OK; /* Return Code */ - LsmPgno iPtr; /* Pointer value to accompany pKey/nKey */ - - assert( pMW->aSave[0].bStore==0 ); - assert( pMW->aSave[1].bStore==0 ); - rc = mergeWorkerBtreeIndirect(pMW); - - /* Obtain the absolute pointer value to store along with the key in the - ** page body. This pointer points to a page that contains keys that are - ** smaller than pKey/nKey. */ - iPtr = pMW->aSave[0].iPgno; - assert( iPtr!=0 ); - - /* Determine if the indirect format should be used. */ - if( (nKey*4 > lsmFsPageSize(pMW->pDb->pFS)) ){ - pMW->iIndirect = iPtr; - pMW->aSave[1].bStore = 1; - }else{ - rc = mergeWorkerBtreeWrite( - pMW, (u8)(iTopic | LSM_SEPARATOR), iPtr, 0, pKey, nKey - ); - } - - /* Ensure that the SortedRun.iRoot field is correct. */ - return rc; -} - -static int mergeWorkerFinishHierarchy( - MergeWorker *pMW /* Merge worker object */ -){ - int i; /* Used to loop through apHier[] */ - int rc = LSM_OK; /* Return code */ - LsmPgno iPtr; /* New right-hand-child pointer value */ - - iPtr = pMW->aSave[0].iPgno; - for(i=0; i<pMW->hier.nHier && rc==LSM_OK; i++){ - Page *pPg = pMW->hier.apHier[i]; - int nData; /* Size of aData[] in bytes */ - u8 *aData; /* Page data for pPg */ - - aData = fsPageData(pPg, &nData); - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); - - rc = lsmFsPagePersist(pPg); - iPtr = lsmFsPageNumber(pPg); - lsmFsPageRelease(pPg); - } - - if( pMW->hier.nHier ){ - pMW->pLevel->lhs.iRoot = iPtr; - lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); - pMW->hier.apHier = 0; - pMW->hier.nHier = 0; - } - - return rc; -} - -static int mergeWorkerAddPadding( - MergeWorker *pMW /* Merge worker object */ -){ - FileSystem *pFS = pMW->pDb->pFS; - return lsmFsSortedPadding(pFS, pMW->pDb->pWorker, &pMW->pLevel->lhs); -} - -/* -** Release all page references currently held by the merge-worker passed -** as the only argument. Unless an error has occurred, all pages have -** already been released. -*/ -static void mergeWorkerReleaseAll(MergeWorker *pMW){ - int i; - lsmFsPageRelease(pMW->pPage); - pMW->pPage = 0; - - for(i=0; i<pMW->hier.nHier; i++){ - lsmFsPageRelease(pMW->hier.apHier[i]); - pMW->hier.apHier[i] = 0; - } - lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); - pMW->hier.apHier = 0; - pMW->hier.nHier = 0; -} - -static int keyszToSkip(FileSystem *pFS, int nKey){ - int nPgsz; /* Nominal database page size */ - nPgsz = lsmFsPageSize(pFS); - return LSM_MIN(((nKey * 4) / nPgsz), 3); -} - -/* -** Release the reference to the current output page of merge-worker *pMW -** (reference pMW->pPage). Set the page number values in aSave[] as -** required (see comments above struct MergeWorker for details). -*/ -static int mergeWorkerPersistAndRelease(MergeWorker *pMW){ - int rc; - int i; - - assert( pMW->pPage || (pMW->aSave[0].bStore==0 && pMW->aSave[1].bStore==0) ); - - /* Persist the page */ - rc = lsmFsPagePersist(pMW->pPage); - - /* If required, save the page number. */ - for(i=0; i<2; i++){ - if( pMW->aSave[i].bStore ){ - pMW->aSave[i].iPgno = lsmFsPageNumber(pMW->pPage); - pMW->aSave[i].bStore = 0; - } - } - - /* Release the completed output page. */ - lsmFsPageRelease(pMW->pPage); - pMW->pPage = 0; - return rc; -} - -/* -** Advance to the next page of an output run being populated by merge-worker -** pMW. The footer of the new page is initialized to indicate that it contains -** zero records. The flags field is cleared. The page footer pointer field -** is set to iFPtr. -** -** If successful, LSM_OK is returned. Otherwise, an error code. -*/ -static int mergeWorkerNextPage( - MergeWorker *pMW, /* Merge worker object to append page to */ - LsmPgno iFPtr /* Pointer value for footer of new page */ -){ - int rc = LSM_OK; /* Return code */ - Page *pNext = 0; /* New page appended to run */ - lsm_db *pDb = pMW->pDb; /* Database handle */ - - rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext); - assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress ); - - if( rc==LSM_OK ){ - u8 *aData; /* Data buffer belonging to page pNext */ - int nData; /* Size of aData[] in bytes */ - - rc = mergeWorkerPersistAndRelease(pMW); - - pMW->pPage = pNext; - pMW->pLevel->pMerge->iOutputOff = 0; - aData = fsPageData(pNext, &nData); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], 0); - lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], 0); - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iFPtr); - pMW->nWork++; - } - - return rc; -} - -/* -** Write a blob of data into an output segment being populated by a -** merge-worker object. If argument bSep is true, write into the separators -** array. Otherwise, the main array. -** -** This function is used to write the blobs of data for keys and values. -*/ -static int mergeWorkerData( - MergeWorker *pMW, /* Merge worker object */ - int bSep, /* True to write to separators run */ - LsmPgno iFPtr, /* Footer ptr for new pages */ - u8 *aWrite, /* Write data from this buffer */ - int nWrite /* Size of aWrite[] in bytes */ -){ - int rc = LSM_OK; /* Return code */ - int nRem = nWrite; /* Number of bytes still to write */ - - while( rc==LSM_OK && nRem>0 ){ - Merge *pMerge = pMW->pLevel->pMerge; - int nCopy; /* Number of bytes to copy */ - u8 *aData; /* Pointer to buffer of current output page */ - int nData; /* Size of aData[] in bytes */ - int nRec; /* Number of records on current output page */ - int iOff; /* Offset in aData[] to write to */ - - assert( lsmFsPageWritable(pMW->pPage) ); - - aData = fsPageData(pMW->pPage, &nData); - nRec = pageGetNRec(aData, nData); - iOff = pMerge->iOutputOff; - nCopy = LSM_MIN(nRem, SEGMENT_EOF(nData, nRec) - iOff); - - memcpy(&aData[iOff], &aWrite[nWrite-nRem], nCopy); - nRem -= nCopy; - - if( nRem>0 ){ - rc = mergeWorkerNextPage(pMW, iFPtr); - }else{ - pMerge->iOutputOff = iOff + nCopy; - } - } - - return rc; -} - - -/* -** The MergeWorker passed as the only argument is working to merge two or -** more existing segments together (not to flush an in-memory tree). It -** has not yet written the first key to the first page of the output. -*/ -static int mergeWorkerFirstPage(MergeWorker *pMW){ - int rc = LSM_OK; /* Return code */ - Page *pPg = 0; /* First page of run pSeg */ - LsmPgno iFPtr = 0; /* Pointer value read from footer of pPg */ - MultiCursor *pCsr = pMW->pCsr; - - assert( pMW->pPage==0 ); - - if( pCsr->pBtCsr ){ - rc = LSM_OK; - iFPtr = pMW->pLevel->pNext->lhs.iFirst; - }else if( pCsr->nPtr>0 ){ - Segment *pSeg; - pSeg = pCsr->aPtr[pCsr->nPtr-1].pSeg; - rc = lsmFsDbPageGet(pMW->pDb->pFS, pSeg, pSeg->iFirst, &pPg); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer for page pPg */ - int nData; /* Size of aData[] in bytes */ - aData = fsPageData(pPg, &nData); - iFPtr = pageGetPtr(aData, nData); - lsmFsPageRelease(pPg); - } - } - - if( rc==LSM_OK ){ - rc = mergeWorkerNextPage(pMW, iFPtr); - if( pCsr->pPrevMergePtr ) *pCsr->pPrevMergePtr = iFPtr; - pMW->aSave[0].bStore = 1; - } - - return rc; -} - -static int mergeWorkerWrite( - MergeWorker *pMW, /* Merge worker object to write into */ - int eType, /* One of SORTED_SEPARATOR, WRITE or DELETE */ - void *pKey, int nKey, /* Key value */ - void *pVal, int nVal, /* Value value */ - LsmPgno iPtr /* Absolute value of page pointer, or 0 */ -){ - int rc = LSM_OK; /* Return code */ - Merge *pMerge; /* Persistent part of level merge state */ - int nHdr; /* Space required for this record header */ - Page *pPg; /* Page to write to */ - u8 *aData; /* Data buffer for page pWriter->pPage */ - int nData = 0; /* Size of buffer aData[] in bytes */ - int nRec = 0; /* Number of records on page pPg */ - LsmPgno iFPtr = 0; /* Value of pointer in footer of pPg */ - LsmPgno iRPtr = 0; /* Value of pointer written into record */ - int iOff = 0; /* Current write offset within page pPg */ - Segment *pSeg; /* Segment being written */ - int flags = 0; /* If != 0, flags value for page footer */ - int bFirst = 0; /* True for first key of output run */ - - pMerge = pMW->pLevel->pMerge; - pSeg = &pMW->pLevel->lhs; - - if( pSeg->iFirst==0 && pMW->pPage==0 ){ - rc = mergeWorkerFirstPage(pMW); - bFirst = 1; - } - pPg = pMW->pPage; - if( pPg ){ - aData = fsPageData(pPg, &nData); - nRec = pageGetNRec(aData, nData); - iFPtr = pageGetPtr(aData, nData); - iRPtr = iPtr ? (iPtr - iFPtr) : 0; - } - - /* Figure out how much space is required by the new record. The space - ** required is divided into two sections: the header and the body. The - ** header consists of the intial varint fields. The body are the blobs - ** of data that correspond to the key and value data. The entire header - ** must be stored on the page. The body may overflow onto the next and - ** subsequent pages. - ** - ** The header space is: - ** - ** 1) record type - 1 byte. - ** 2) Page-pointer-offset - 1 varint - ** 3) Key size - 1 varint - ** 4) Value size - 1 varint (only if LSM_INSERT flag is set) - */ - if( rc==LSM_OK ){ - nHdr = 1 + lsmVarintLen64(iRPtr) + lsmVarintLen32(nKey); - if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal); - - /* If the entire header will not fit on page pPg, or if page pPg is - ** marked read-only, advance to the next page of the output run. */ - iOff = pMerge->iOutputOff; - if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){ - if( iOff>=0 && pPg ){ - /* Zero any free space on the page */ - assert( aData ); - memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff); - } - iFPtr = *pMW->pCsr->pPrevMergePtr; - iRPtr = iPtr ? (iPtr - iFPtr) : 0; - iOff = 0; - nRec = 0; - rc = mergeWorkerNextPage(pMW, iFPtr); - pPg = pMW->pPage; - } - } - - /* If this record header will be the first on the page, and the page is - ** not the very first in the entire run, add a copy of the key to the - ** b-tree hierarchy. - */ - if( rc==LSM_OK && nRec==0 && bFirst==0 ){ - assert( pMerge->nSkip>=0 ); - - if( pMerge->nSkip==0 ){ - rc = mergeWorkerPushHierarchy(pMW, rtTopic(eType), pKey, nKey); - assert( pMW->aSave[0].bStore==0 ); - pMW->aSave[0].bStore = 1; - pMerge->nSkip = keyszToSkip(pMW->pDb->pFS, nKey); - }else{ - pMerge->nSkip--; - flags = PGFTR_SKIP_THIS_FLAG; - } - - if( pMerge->nSkip ) flags |= PGFTR_SKIP_NEXT_FLAG; - } - - /* Update the output segment */ - if( rc==LSM_OK ){ - aData = fsPageData(pPg, &nData); - - /* Update the page footer. */ - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1)); - lsmPutU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec)], (u16)iOff); - if( flags ) lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], (u16)flags); - - /* Write the entry header into the current page. */ - aData[iOff++] = (u8)eType; /* 1 */ - iOff += lsmVarintPut64(&aData[iOff], iRPtr); /* 2 */ - iOff += lsmVarintPut32(&aData[iOff], nKey); /* 3 */ - if( rtIsWrite(eType) ) iOff += lsmVarintPut32(&aData[iOff], nVal); /* 4 */ - pMerge->iOutputOff = iOff; - - /* Write the key and data into the segment. */ - assert( iFPtr==pageGetPtr(aData, nData) ); - rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pKey, nKey); - if( rc==LSM_OK && rtIsWrite(eType) ){ - if( rc==LSM_OK ){ - rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pVal, nVal); - } - } - } - - return rc; -} - - -/* -** Free all resources allocated by mergeWorkerInit(). -*/ -static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){ - int i; /* Iterator variable */ - int rc = *pRc; - MultiCursor *pCsr = pMW->pCsr; - - /* Unless the merge has finished, save the cursor position in the - ** Merge.aInput[] array. See function mergeWorkerInit() for the - ** code to restore a cursor position based on aInput[]. */ - if( rc==LSM_OK && pCsr ){ - Merge *pMerge = pMW->pLevel->pMerge; - if( lsmMCursorValid(pCsr) ){ - int bBtree = (pCsr->pBtCsr!=0); - int iPtr; - - /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */ - assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 ); - assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) ); - - for(i=0; i<(pMerge->nInput-bBtree); i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - if( pPtr->pPg ){ - pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg); - pMerge->aInput[i].iCell = pPtr->iCell; - }else{ - pMerge->aInput[i].iPg = 0; - pMerge->aInput[i].iCell = 0; - } - } - if( bBtree && pMerge->nInput ){ - assert( i==pCsr->nPtr ); - btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]); - } - - /* Store the location of the split-key */ - iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iPtr<pCsr->nPtr ){ - pMerge->splitkey = pMerge->aInput[iPtr]; - }else{ - btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey); - } - } - - /* Zero any free space left on the final page. This helps with - ** compression if using a compression hook. And prevents valgrind - ** from complaining about uninitialized byte passed to write(). */ - if( pMW->pPage ){ - int nData; - u8 *aData = fsPageData(pMW->pPage, &nData); - int iOff = pMerge->iOutputOff; - int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData)); - memset(&aData[iOff], 0, iEof - iOff); - } - - pMerge->iOutputOff = -1; - } - - lsmMCursorClose(pCsr, 0); - - /* Persist and release the output page. */ - if( rc==LSM_OK ) rc = mergeWorkerPersistAndRelease(pMW); - if( rc==LSM_OK ) rc = mergeWorkerBtreeIndirect(pMW); - if( rc==LSM_OK ) rc = mergeWorkerFinishHierarchy(pMW); - if( rc==LSM_OK ) rc = mergeWorkerAddPadding(pMW); - lsmFsFlushWaiting(pMW->pDb->pFS, &rc); - mergeWorkerReleaseAll(pMW); - - lsmFree(pMW->pDb->pEnv, pMW->aGobble); - pMW->aGobble = 0; - pMW->pCsr = 0; - - *pRc = rc; -} - -/* -** The cursor passed as the first argument is being used as the input for -** a merge operation. When this function is called, *piFlags contains the -** database entry flags for the current entry. The entry about to be written -** to the output. -** -** Note that this function only has to work for cursors configured to -** iterate forwards (not backwards). -*/ -static void mergeRangeDeletes(MultiCursor *pCsr, int *piVal, int *piFlags){ - int f = *piFlags; - int iKey = pCsr->aTree[1]; - int i; - - assert( pCsr->flags & CURSOR_NEXT_OK ); - if( pCsr->flags & CURSOR_IGNORE_DELETE ){ - /* The ignore-delete flag is set when the output of the merge will form - ** the oldest level in the database. In this case there is no point in - ** retaining any range-delete flags. */ - assert( (f & LSM_POINT_DELETE)==0 ); - f &= ~(LSM_START_DELETE|LSM_END_DELETE); - }else{ - for(i=0; i<(CURSOR_DATA_SEGMENT + pCsr->nPtr); i++){ - if( i!=iKey ){ - int eType; - void *pKey; - int nKey; - int res; - multiCursorGetKey(pCsr, i, &eType, &pKey, &nKey); - - if( pKey ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, - rtTopic(eType), pKey, nKey - ); - assert( res<=0 ); - if( res==0 ){ - if( (f & (LSM_INSERT|LSM_POINT_DELETE))==0 ){ - if( eType & LSM_INSERT ){ - f |= LSM_INSERT; - *piVal = i; - } - else if( eType & LSM_POINT_DELETE ){ - f |= LSM_POINT_DELETE; - } - } - f |= (eType & (LSM_END_DELETE|LSM_START_DELETE)); - } - - if( i>iKey && (eType & LSM_END_DELETE) && res<0 ){ - if( f & (LSM_INSERT|LSM_POINT_DELETE) ){ - f |= (LSM_END_DELETE|LSM_START_DELETE); - }else{ - f = 0; - } - break; - } - } - } - } - - assert( (f & LSM_INSERT)==0 || (f & LSM_POINT_DELETE)==0 ); - if( (f & LSM_START_DELETE) - && (f & LSM_END_DELETE) - && (f & LSM_POINT_DELETE ) - ){ - f = 0; - } - } - - *piFlags = f; -} - -static int mergeWorkerStep(MergeWorker *pMW){ - lsm_db *pDb = pMW->pDb; /* Database handle */ - MultiCursor *pCsr; /* Cursor to read input data from */ - int rc = LSM_OK; /* Return code */ - int eType; /* SORTED_SEPARATOR, WRITE or DELETE */ - void *pKey; int nKey; /* Key */ - LsmPgno iPtr; - int iVal; - - pCsr = pMW->pCsr; - - /* Pull the next record out of the source cursor. */ - lsmMCursorKey(pCsr, &pKey, &nKey); - eType = pCsr->eType; - - /* Figure out if the output record may have a different pointer value - ** than the previous. This is the case if the current key is identical to - ** a key that appears in the lowest level run being merged. If so, set - ** iPtr to the absolute pointer value. If not, leave iPtr set to zero, - ** indicating that the output pointer value should be a copy of the pointer - ** value written with the previous key. */ - iPtr = (pCsr->pPrevMergePtr ? *pCsr->pPrevMergePtr : 0); - if( pCsr->pBtCsr ){ - BtreeCursor *pBtCsr = pCsr->pBtCsr; - if( pBtCsr->pKey ){ - int res = rtTopic(pBtCsr->eType) - rtTopic(eType); - if( res==0 ) res = pDb->xCmp(pBtCsr->pKey, pBtCsr->nKey, pKey, nKey); - if( 0==res ) iPtr = pBtCsr->iPtr; - assert( res>=0 ); - } - }else if( pCsr->nPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[pCsr->nPtr-1]; - if( pPtr->pPg - && 0==pDb->xCmp(pPtr->pKey, pPtr->nKey, pKey, nKey) - ){ - iPtr = pPtr->iPtr+pPtr->iPgPtr; - } - } - - iVal = pCsr->aTree[1]; - mergeRangeDeletes(pCsr, &iVal, &eType); - - if( eType!=0 ){ - if( pMW->aGobble ){ - int iGobble = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iGobble<pCsr->nPtr && iGobble>=0 ){ - SegmentPtr *pGobble = &pCsr->aPtr[iGobble]; - if( (pGobble->flags & PGFTR_SKIP_THIS_FLAG)==0 ){ - pMW->aGobble[iGobble] = lsmFsPageNumber(pGobble->pPg); - } - } - } - - /* If this is a separator key and we know that the output pointer has not - ** changed, there is no point in writing an output record. Otherwise, - ** proceed. */ - if( rc==LSM_OK && (rtIsSeparator(eType)==0 || iPtr!=0) ){ - /* Write the record into the main run. */ - void *pVal; int nVal; - rc = multiCursorGetVal(pCsr, iVal, &pVal, &nVal); - if( pVal && rc==LSM_OK ){ - assert( nVal>=0 ); - rc = sortedBlobSet(pDb->pEnv, &pCsr->val, pVal, nVal); - pVal = pCsr->val.pData; - } - if( rc==LSM_OK ){ - rc = mergeWorkerWrite(pMW, eType, pKey, nKey, pVal, nVal, iPtr); - } - } - } - - /* Advance the cursor to the next input record (assuming one exists). */ - assert( lsmMCursorValid(pMW->pCsr) ); - if( rc==LSM_OK ) rc = lsmMCursorNext(pMW->pCsr); - - return rc; -} - -static int mergeWorkerDone(MergeWorker *pMW){ - return pMW->pCsr==0 || !lsmMCursorValid(pMW->pCsr); -} - -static void sortedFreeLevel(lsm_env *pEnv, Level *p){ - if( p ){ - lsmFree(pEnv, p->pSplitKey); - lsmFree(pEnv, p->pMerge); - lsmFree(pEnv, p->aRhs); - lsmFree(pEnv, p); - } -} - -static void sortedInvokeWorkHook(lsm_db *pDb){ - if( pDb->xWork ){ - pDb->xWork(pDb, pDb->pWorkCtx); - } -} - -static int sortedNewToplevel( - lsm_db *pDb, /* Connection handle */ - int eTree, /* One of the TREE_XXX constants */ - int *pnWrite /* OUT: Number of database pages written */ -){ - int rc = LSM_OK; /* Return Code */ - MultiCursor *pCsr = 0; - Level *pNext = 0; /* The current top level */ - Level *pNew; /* The new level itself */ - Segment *pLinked = 0; /* Delete separators from this segment */ - Level *pDel = 0; /* Delete this entire level */ - int nWrite = 0; /* Number of database pages written */ - Freelist freelist; - - if( eTree!=TREE_NONE ){ - rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); - } - - assert( pDb->bUseFreelist==0 ); - pDb->pFreelist = &freelist; - pDb->bUseFreelist = 1; - memset(&freelist, 0, sizeof(freelist)); - - /* Allocate the new level structure to write to. */ - pNext = lsmDbSnapshotLevel(pDb->pWorker); - pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); - if( pNew ){ - pNew->pNext = pNext; - lsmDbSnapshotSetLevel(pDb->pWorker, pNew); - } - - /* Create a cursor to gather the data required by the new segment. The new - ** segment contains everything in the tree and pointers to the next segment - ** in the database (if any). */ - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - pCsr->pDb = pDb; - rc = multiCursorVisitFreelist(pCsr); - if( rc==LSM_OK ){ - rc = multiCursorAddTree(pCsr, pDb->pWorker, eTree); - } - if( rc==LSM_OK && pNext && pNext->pMerge==0 ){ - if( (pNext->flags & LEVEL_FREELIST_ONLY) ){ - pDel = pNext; - pCsr->aPtr = lsmMallocZeroRc(pDb->pEnv, sizeof(SegmentPtr), &rc); - multiCursorAddOne(pCsr, pNext, &rc); - }else if( eTree!=TREE_NONE && pNext->lhs.iRoot ){ - pLinked = &pNext->lhs; - rc = btreeCursorNew(pDb, pLinked, &pCsr->pBtCsr); - } - } - - /* If this will be the only segment in the database, discard any delete - ** markers present in the in-memory tree. */ - if( pNext==0 ){ - multiCursorIgnoreDelete(pCsr); - } - } - - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - }else{ - LsmPgno iLeftPtr = 0; - Merge merge; /* Merge object used to create new level */ - MergeWorker mergeworker; /* MergeWorker object for the same purpose */ - - memset(&merge, 0, sizeof(Merge)); - memset(&mergeworker, 0, sizeof(MergeWorker)); - - pNew->pMerge = &merge; - pNew->flags |= LEVEL_INCOMPLETE; - mergeworker.pDb = pDb; - mergeworker.pLevel = pNew; - mergeworker.pCsr = pCsr; - pCsr->pPrevMergePtr = &iLeftPtr; - - /* Mark the separators array for the new level as a "phantom". */ - mergeworker.bFlush = 1; - - /* Do the work to create the new merged segment on disk */ - if( rc==LSM_OK ) rc = lsmMCursorFirst(pCsr); - while( rc==LSM_OK && mergeWorkerDone(&mergeworker)==0 ){ - rc = mergeWorkerStep(&mergeworker); - } - mergeWorkerShutdown(&mergeworker, &rc); - assert( rc!=LSM_OK || mergeworker.nWork==0 || pNew->lhs.iFirst ); - if( rc==LSM_OK && pNew->lhs.iFirst ){ - rc = lsmFsSortedFinish(pDb->pFS, &pNew->lhs); - } - nWrite = mergeworker.nWork; - pNew->flags &= ~LEVEL_INCOMPLETE; - if( eTree==TREE_NONE ){ - pNew->flags |= LEVEL_FREELIST_ONLY; - } - pNew->pMerge = 0; - } - - if( rc!=LSM_OK || pNew->lhs.iFirst==0 ){ - assert( rc!=LSM_OK || pDb->pWorker->freelist.nEntry==0 ); - lsmDbSnapshotSetLevel(pDb->pWorker, pNext); - sortedFreeLevel(pDb->pEnv, pNew); - }else{ - if( pLinked ){ - pLinked->iRoot = 0; - }else if( pDel ){ - assert( pNew->pNext==pDel ); - pNew->pNext = pDel->pNext; - lsmFsSortedDelete(pDb->pFS, pDb->pWorker, 1, &pDel->lhs); - sortedFreeLevel(pDb->pEnv, pDel); - } - -#if LSM_LOG_STRUCTURE - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "new-toplevel"); -#endif - - if( freelist.nEntry ){ - Freelist *p = &pDb->pWorker->freelist; - lsmFree(pDb->pEnv, p->aEntry); - memcpy(p, &freelist, sizeof(freelist)); - freelist.aEntry = 0; - }else{ - pDb->pWorker->freelist.nEntry = 0; - } - - assertBtreeOk(pDb, &pNew->lhs); - sortedInvokeWorkHook(pDb); - } - - if( pnWrite ) *pnWrite = nWrite; - pDb->pWorker->nWrite += nWrite; - pDb->pFreelist = 0; - pDb->bUseFreelist = 0; - lsmFree(pDb->pEnv, freelist.aEntry); - return rc; -} - -/* -** The nMerge levels in the LSM beginning with pLevel consist of a -** left-hand-side segment only. Replace these levels with a single new -** level consisting of a new empty segment on the left-hand-side and the -** nMerge segments from the replaced levels on the right-hand-side. -** -** Also, allocate and populate a Merge object and set Level.pMerge to -** point to it. -*/ -static int sortedMergeSetup( - lsm_db *pDb, /* Database handle */ - Level *pLevel, /* First level to merge */ - int nMerge, /* Merge this many levels together */ - Level **ppNew /* New, merged, level */ -){ - int rc = LSM_OK; /* Return Code */ - Level *pNew; /* New Level object */ - int bUseNext = 0; /* True to link in next separators */ - Merge *pMerge; /* New Merge object */ - int nByte; /* Bytes of space allocated at pMerge */ - -#ifdef LSM_DEBUG - int iLevel; - Level *pX = pLevel; - for(iLevel=0; iLevel<nMerge; iLevel++){ - assert( pX->nRight==0 ); - pX = pX->pNext; - } -#endif - - /* Allocate the new Level object */ - pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); - if( pNew ){ - pNew->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, - nMerge * sizeof(Segment), &rc); - } - - /* Populate the new Level object */ - if( rc==LSM_OK ){ - Level *pNext = 0; /* Level following pNew */ - int i; - int bFreeOnly = 1; - Level *pTopLevel; - Level *p = pLevel; - Level **pp; - pNew->nRight = nMerge; - pNew->iAge = pLevel->iAge+1; - for(i=0; i<nMerge; i++){ - assert( p->nRight==0 ); - pNext = p->pNext; - pNew->aRhs[i] = p->lhs; - if( (p->flags & LEVEL_FREELIST_ONLY)==0 ) bFreeOnly = 0; - sortedFreeLevel(pDb->pEnv, p); - p = pNext; - } - - if( bFreeOnly ) pNew->flags |= LEVEL_FREELIST_ONLY; - - /* Replace the old levels with the new. */ - pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - pNew->pNext = p; - for(pp=&pTopLevel; *pp!=pLevel; pp=&((*pp)->pNext)); - *pp = pNew; - lsmDbSnapshotSetLevel(pDb->pWorker, pTopLevel); - - /* Determine whether or not the next separators will be linked in */ - if( pNext && pNext->pMerge==0 && pNext->lhs.iRoot && pNext - && (bFreeOnly==0 || (pNext->flags & LEVEL_FREELIST_ONLY)) - ){ - bUseNext = 1; - } - } - - /* Allocate the merge object */ - nByte = sizeof(Merge) + sizeof(MergeInput) * (nMerge + bUseNext); - pMerge = (Merge *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - if( pMerge ){ - pMerge->aInput = (MergeInput *)&pMerge[1]; - pMerge->nInput = nMerge + bUseNext; - pNew->pMerge = pMerge; - } - - *ppNew = pNew; - return rc; -} - -static int mergeWorkerInit( - lsm_db *pDb, /* Db connection to do merge work */ - Level *pLevel, /* Level to work on merging */ - MergeWorker *pMW /* Object to initialize */ -){ - int rc = LSM_OK; /* Return code */ - Merge *pMerge = pLevel->pMerge; /* Persistent part of merge state */ - MultiCursor *pCsr = 0; /* Cursor opened for pMW */ - Level *pNext = pLevel->pNext; /* Next level in LSM */ - - assert( pDb->pWorker ); - assert( pLevel->pMerge ); - assert( pLevel->nRight>0 ); - - memset(pMW, 0, sizeof(MergeWorker)); - pMW->pDb = pDb; - pMW->pLevel = pLevel; - pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc); - - /* Create a multi-cursor to read the data to write to the new - ** segment. The new segment contains: - ** - ** 1. Records from LHS of each of the nMerge levels being merged. - ** 2. Separators from either the last level being merged, or the - ** separators attached to the LHS of the following level, or neither. - ** - ** If the new level is the lowest (oldest) in the db, discard any - ** delete keys. Key annihilation. - */ - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - pCsr->flags |= CURSOR_NEXT_OK; - rc = multiCursorAddRhs(pCsr, pLevel); - } - if( rc==LSM_OK && pMerge->nInput > pLevel->nRight ){ - rc = btreeCursorNew(pDb, &pNext->lhs, &pCsr->pBtCsr); - }else if( pNext ){ - multiCursorReadSeparators(pCsr); - }else{ - multiCursorIgnoreDelete(pCsr); - } - - assert( rc!=LSM_OK || pMerge->nInput==(pCsr->nPtr+(pCsr->pBtCsr!=0)) ); - pMW->pCsr = pCsr; - - /* Load the b-tree hierarchy into memory. */ - if( rc==LSM_OK ) rc = mergeWorkerLoadHierarchy(pMW); - if( rc==LSM_OK && pMW->hier.nHier==0 ){ - pMW->aSave[0].iPgno = pLevel->lhs.iFirst; - } - - /* Position the cursor. */ - if( rc==LSM_OK ){ - pCsr->pPrevMergePtr = &pMerge->iCurrentPtr; - if( pLevel->lhs.iFirst==0 ){ - /* The output array is still empty. So position the cursor at the very - ** start of the input. */ - rc = multiCursorEnd(pCsr, 0); - }else{ - /* The output array is non-empty. Position the cursor based on the - ** page/cell data saved in the Merge.aInput[] array. */ - int i; - for(i=0; rc==LSM_OK && i<pCsr->nPtr; i++){ - MergeInput *pInput = &pMerge->aInput[i]; - if( pInput->iPg ){ - SegmentPtr *pPtr; - assert( pCsr->aPtr[i].pPg==0 ); - pPtr = &pCsr->aPtr[i]; - rc = segmentPtrLoadPage(pDb->pFS, pPtr, pInput->iPg); - if( rc==LSM_OK && pPtr->nCell>0 ){ - rc = segmentPtrLoadCell(pPtr, pInput->iCell); - } - } - } - - if( rc==LSM_OK && pCsr->pBtCsr ){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - assert( i==pCsr->nPtr ); - rc = btreeCursorRestore(pCsr->pBtCsr, xCmp, &pMerge->aInput[i]); - } - - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(pCsr, 0); - } - } - pCsr->flags |= CURSOR_NEXT_OK; - } - - return rc; -} - -static int sortedBtreeGobble( - lsm_db *pDb, /* Worker connection */ - MultiCursor *pCsr, /* Multi-cursor being used for a merge */ - int iGobble /* pCsr->aPtr[] entry to operate on */ -){ - int rc = LSM_OK; - if( rtTopic(pCsr->eType)==0 ){ - Segment *pSeg = pCsr->aPtr[iGobble].pSeg; - LsmPgno *aPg; - int nPg; - - /* Seek from the root of the b-tree to the segment leaf that may contain - ** a key equal to the one multi-cursor currently points to. Record the - ** page number of each b-tree page and the leaf. The segment may be - ** gobbled up to (but not including) the first of these page numbers. - */ - assert( pSeg->iRoot>0 ); - aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc); - if( rc==LSM_OK ){ - rc = seekInBtree(pCsr, pSeg, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0 - ); - } - - if( rc==LSM_OK ){ - for(nPg=0; aPg[nPg]; nPg++); - lsmFsGobble(pDb, pSeg, aPg, nPg); - } - - lsmFree(pDb->pEnv, aPg); - } - return rc; -} - -/* -** Argument p points to a level of age N. Return the number of levels in -** the linked list starting at p that have age=N (always at least 1). -*/ -static int sortedCountLevels(Level *p){ - int iAge = p->iAge; - int nRet = 0; - do { - nRet++; - p = p->pNext; - }while( p && p->iAge==iAge ); - return nRet; -} - -static int sortedSelectLevel(lsm_db *pDb, int nMerge, Level **ppOut){ - Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - int rc = LSM_OK; - Level *pLevel = 0; /* Output value */ - Level *pBest = 0; /* Best level to work on found so far */ - int nBest; /* Number of segments merged at pBest */ - Level *pThis = 0; /* First in run of levels with age=iAge */ - int nThis = 0; /* Number of levels starting at pThis */ - - assert( nMerge>=1 ); - nBest = LSM_MAX(1, nMerge-1); - - /* Find the longest contiguous run of levels not currently undergoing a - ** merge with the same age in the structure. Or the level being merged - ** with the largest number of right-hand segments. Work on it. */ - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - if( pLevel->nRight==0 && pThis && pLevel->iAge==pThis->iAge ){ - nThis++; - }else{ - if( nThis>nBest ){ - if( (pLevel->iAge!=pThis->iAge+1) - || (pLevel->nRight==0 && sortedCountLevels(pLevel)<=pDb->nMerge) - ){ - pBest = pThis; - nBest = nThis; - } - } - if( pLevel->nRight ){ - if( pLevel->nRight>nBest ){ - nBest = pLevel->nRight; - pBest = pLevel; - } - nThis = 0; - pThis = 0; - }else{ - pThis = pLevel; - nThis = 1; - } - } - } - if( nThis>nBest ){ - assert( pThis ); - pBest = pThis; - nBest = nThis; - } - - if( pBest==0 && nMerge==1 ){ - int nFree = 0; - int nUsr = 0; - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - assert( !pLevel->nRight ); - if( pLevel->flags & LEVEL_FREELIST_ONLY ){ - nFree++; - }else{ - nUsr++; - } - } - if( nUsr>1 ){ - pBest = pTopLevel; - nBest = nFree + nUsr; - } - } - - if( pBest ){ - if( pBest->nRight==0 ){ - rc = sortedMergeSetup(pDb, pBest, nBest, ppOut); - }else{ - *ppOut = pBest; - } - } - - return rc; -} - -static int sortedDbIsFull(lsm_db *pDb){ - Level *pTop = lsmDbSnapshotLevel(pDb->pWorker); - - if( lsmDatabaseFull(pDb) ) return 1; - if( pTop && pTop->iAge==0 - && (pTop->nRight || sortedCountLevels(pTop)>=pDb->nMerge) - ){ - return 1; - } - return 0; -} - -typedef struct MoveBlockCtx MoveBlockCtx; -struct MoveBlockCtx { - int iSeen; /* Previous free block on list */ - int iFrom; /* Total number of blocks in file */ -}; - -static int moveBlockCb(void *pCtx, int iBlk, i64 iSnapshot){ - MoveBlockCtx *p = (MoveBlockCtx *)pCtx; - assert( p->iFrom==0 ); - if( iBlk==(p->iSeen-1) ){ - p->iSeen = iBlk; - return 0; - } - p->iFrom = p->iSeen-1; - return 1; -} - -/* -** This function is called to further compact a database for which all -** of the content has already been merged into a single segment. If -** possible, it moves the contents of a single block from the end of the -** file to a free-block that lies closer to the start of the file (allowing -** the file to be eventually truncated). -*/ -static int sortedMoveBlock(lsm_db *pDb, int *pnWrite){ - Snapshot *p = pDb->pWorker; - Level *pLvl = lsmDbSnapshotLevel(p); - int iFrom; /* Block to move */ - int iTo; /* Destination to move block to */ - int rc; /* Return code */ - - MoveBlockCtx sCtx; - - assert( pLvl->pNext==0 && pLvl->nRight==0 ); - assert( p->redirect.n<=LSM_MAX_BLOCK_REDIRECTS ); - - *pnWrite = 0; - - /* Check that the redirect array is not already full. If it is, return - ** without moving any database content. */ - if( p->redirect.n>=LSM_MAX_BLOCK_REDIRECTS ) return LSM_OK; - - /* Find the last block of content in the database file. Do this by - ** traversing the free-list in reverse (descending block number) order. - ** The first block not on the free list is the one that will be moved. - ** Since the db consists of a single segment, there is no ambiguity as - ** to which segment the block belongs to. */ - sCtx.iSeen = p->nBlock+1; - sCtx.iFrom = 0; - rc = lsmWalkFreelist(pDb, 1, moveBlockCb, &sCtx); - if( rc!=LSM_OK || sCtx.iFrom==0 ) return rc; - iFrom = sCtx.iFrom; - - /* Find the first free block in the database, ignoring block 1. Block - ** 1 is tricky as it is smaller than the other blocks. */ - rc = lsmBlockAllocate(pDb, iFrom, &iTo); - if( rc!=LSM_OK || iTo==0 ) return rc; - assert( iTo!=1 && iTo<iFrom ); - - rc = lsmFsMoveBlock(pDb->pFS, &pLvl->lhs, iTo, iFrom); - if( rc==LSM_OK ){ - if( p->redirect.a==0 ){ - int nByte = sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS; - p->redirect.a = lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - } - if( rc==LSM_OK ){ - - /* Check if the block just moved was already redirected. */ - int i; - for(i=0; i<p->redirect.n; i++){ - if( p->redirect.a[i].iTo==iFrom ) break; - } - - if( i==p->redirect.n ){ - /* Block iFrom was not already redirected. Add a new array entry. */ - memmove(&p->redirect.a[1], &p->redirect.a[0], - sizeof(struct RedirectEntry) * p->redirect.n - ); - p->redirect.a[0].iFrom = iFrom; - p->redirect.a[0].iTo = iTo; - p->redirect.n++; - }else{ - /* Block iFrom was already redirected. Overwrite existing entry. */ - p->redirect.a[i].iTo = iTo; - } - - rc = lsmBlockFree(pDb, iFrom); - - *pnWrite = lsmFsBlockSize(pDb->pFS) / lsmFsPageSize(pDb->pFS); - pLvl->lhs.pRedirect = &p->redirect; - } - } - -#if LSM_LOG_STRUCTURE - if( rc==LSM_OK ){ - char aBuf[64]; - sprintf(aBuf, "move-block %d/%d", p->redirect.n-1, LSM_MAX_BLOCK_REDIRECTS); - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, aBuf); - } -#endif - return rc; -} - -/* -*/ -static int mergeInsertFreelistSegments( - lsm_db *pDb, - int nFree, - MergeWorker *pMW -){ - int rc = LSM_OK; - if( nFree>0 ){ - MultiCursor *pCsr = pMW->pCsr; - Level *pLvl = pMW->pLevel; - SegmentPtr *aNew1; - Segment *aNew2; - - Level *pIter; - Level *pNext; - int i = 0; - - aNew1 = (SegmentPtr *)lsmMallocZeroRc( - pDb->pEnv, sizeof(SegmentPtr) * (pCsr->nPtr+nFree), &rc - ); - if( rc ) return rc; - memcpy(&aNew1[nFree], pCsr->aPtr, sizeof(SegmentPtr)*pCsr->nPtr); - pCsr->nPtr += nFree; - lsmFree(pDb->pEnv, pCsr->aTree); - lsmFree(pDb->pEnv, pCsr->aPtr); - pCsr->aTree = 0; - pCsr->aPtr = aNew1; - - aNew2 = (Segment *)lsmMallocZeroRc( - pDb->pEnv, sizeof(Segment) * (pLvl->nRight+nFree), &rc - ); - if( rc ) return rc; - memcpy(&aNew2[nFree], pLvl->aRhs, sizeof(Segment)*pLvl->nRight); - pLvl->nRight += nFree; - lsmFree(pDb->pEnv, pLvl->aRhs); - pLvl->aRhs = aNew2; - - for(pIter=pDb->pWorker->pLevel; rc==LSM_OK && pIter!=pLvl; pIter=pNext){ - Segment *pSeg = &pLvl->aRhs[i]; - memcpy(pSeg, &pIter->lhs, sizeof(Segment)); - - pCsr->aPtr[i].pSeg = pSeg; - pCsr->aPtr[i].pLevel = pLvl; - rc = segmentPtrEnd(pCsr, &pCsr->aPtr[i], 0); - - pDb->pWorker->pLevel = pNext = pIter->pNext; - sortedFreeLevel(pDb->pEnv, pIter); - i++; - } - assert( i==nFree ); - assert( rc!=LSM_OK || pDb->pWorker->pLevel==pLvl ); - - for(i=nFree; i<pCsr->nPtr; i++){ - pCsr->aPtr[i].pSeg = &pLvl->aRhs[i]; - } - - lsmFree(pDb->pEnv, pMW->aGobble); - pMW->aGobble = 0; - } - return rc; -} - -static int sortedWork( - lsm_db *pDb, /* Database handle. Must be worker. */ - int nWork, /* Number of pages of work to do */ - int nMerge, /* Try to merge this many levels at once */ - int bFlush, /* Set if call is to make room for a flush */ - int *pnWrite /* OUT: Actual number of pages written */ -){ - int rc = LSM_OK; /* Return Code */ - int nRemaining = nWork; /* Units of work to do before returning */ - Snapshot *pWorker = pDb->pWorker; - - assert( pWorker ); - if( lsmDbSnapshotLevel(pWorker)==0 ) return LSM_OK; - - while( nRemaining>0 ){ - Level *pLevel = 0; - - /* Find a level to work on. */ - rc = sortedSelectLevel(pDb, nMerge, &pLevel); - assert( rc==LSM_OK || pLevel==0 ); - - if( pLevel==0 ){ - int nDone = 0; - Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - if( bFlush==0 && nMerge==1 && pTopLevel && pTopLevel->pNext==0 ){ - rc = sortedMoveBlock(pDb, &nDone); - } - nRemaining -= nDone; - - /* Could not find any work to do. Finished. */ - if( nDone==0 ) break; - }else{ - int bSave = 0; - Freelist freelist = {0, 0, 0}; - MergeWorker mergeworker; /* State used to work on the level merge */ - - assert( pDb->bIncrMerge==0 ); - assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); - - pDb->bIncrMerge = 1; - rc = mergeWorkerInit(pDb, pLevel, &mergeworker); - assert( mergeworker.nWork==0 ); - - while( rc==LSM_OK - && 0==mergeWorkerDone(&mergeworker) - && (mergeworker.nWork<nRemaining || pDb->bUseFreelist) - ){ - int eType = rtTopic(mergeworker.pCsr->eType); - rc = mergeWorkerStep(&mergeworker); - - /* If the cursor now points at the first entry past the end of the - ** user data (i.e. either to EOF or to the first free-list entry - ** that will be added to the run), then check if it is possible to - ** merge in any free-list entries that are either in-memory or in - ** free-list-only blocks. */ - if( rc==LSM_OK && nMerge==1 && eType==0 - && (rtTopic(mergeworker.pCsr->eType) || mergeWorkerDone(&mergeworker)) - ){ - int nFree = 0; /* Number of free-list-only levels to merge */ - Level *pLvl; - assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); - - /* Now check if all levels containing data newer than this one - ** are single-segment free-list only levels. If so, they will be - ** merged in now. */ - for(pLvl=pDb->pWorker->pLevel; - pLvl!=mergeworker.pLevel && (pLvl->flags & LEVEL_FREELIST_ONLY); - pLvl=pLvl->pNext - ){ - assert( pLvl->nRight==0 ); - nFree++; - } - if( pLvl==mergeworker.pLevel ){ - - rc = mergeInsertFreelistSegments(pDb, nFree, &mergeworker); - if( rc==LSM_OK ){ - rc = multiCursorVisitFreelist(mergeworker.pCsr); - } - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(mergeworker.pCsr, 0); - pDb->pFreelist = &freelist; - pDb->bUseFreelist = 1; - } - } - } - } - nRemaining -= LSM_MAX(mergeworker.nWork, 1); - - if( rc==LSM_OK ){ - /* Check if the merge operation is completely finished. If not, - ** gobble up (declare eligible for recycling) any pages from rhs - ** segments for which the content has been completely merged into - ** the lhs of the level. */ - if( mergeWorkerDone(&mergeworker)==0 ){ - int i; - for(i=0; i<pLevel->nRight; i++){ - SegmentPtr *pGobble = &mergeworker.pCsr->aPtr[i]; - if( pGobble->pSeg->iRoot ){ - rc = sortedBtreeGobble(pDb, mergeworker.pCsr, i); - }else if( mergeworker.aGobble[i] ){ - lsmFsGobble(pDb, pGobble->pSeg, &mergeworker.aGobble[i], 1); - } - } - }else{ - int i; - int bEmpty; - mergeWorkerShutdown(&mergeworker, &rc); - bEmpty = (pLevel->lhs.iFirst==0); - - if( bEmpty==0 && rc==LSM_OK ){ - rc = lsmFsSortedFinish(pDb->pFS, &pLevel->lhs); - } - - if( pDb->bUseFreelist ){ - Freelist *p = &pDb->pWorker->freelist; - lsmFree(pDb->pEnv, p->aEntry); - memcpy(p, &freelist, sizeof(freelist)); - pDb->bUseFreelist = 0; - pDb->pFreelist = 0; - bSave = 1; - } - - for(i=0; i<pLevel->nRight; i++){ - lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->aRhs[i]); - } - - if( bEmpty ){ - /* If the new level is completely empty, remove it from the - ** database snapshot. This can only happen if all input keys were - ** annihilated. Since keys are only annihilated if the new level - ** is the last in the linked list (contains the most ancient of - ** database content), this guarantees that pLevel->pNext==0. */ - Level *pTop; /* Top level of worker snapshot */ - Level **pp; /* Read/write iterator for Level.pNext list */ - - assert( pLevel->pNext==0 ); - - /* Remove the level from the worker snapshot. */ - pTop = lsmDbSnapshotLevel(pWorker); - for(pp=&pTop; *pp!=pLevel; pp=&((*pp)->pNext)); - *pp = pLevel->pNext; - lsmDbSnapshotSetLevel(pWorker, pTop); - - /* Free the Level structure. */ - sortedFreeLevel(pDb->pEnv, pLevel); - }else{ - - /* Free the separators of the next level, if required. */ - if( pLevel->pMerge->nInput > pLevel->nRight ){ - assert( pLevel->pNext->lhs.iRoot ); - pLevel->pNext->lhs.iRoot = 0; - } - - /* Zero the right-hand-side of pLevel */ - lsmFree(pDb->pEnv, pLevel->aRhs); - pLevel->nRight = 0; - pLevel->aRhs = 0; - - /* Free the Merge object */ - lsmFree(pDb->pEnv, pLevel->pMerge); - pLevel->pMerge = 0; - } - - if( bSave && rc==LSM_OK ){ - pDb->bIncrMerge = 0; - rc = lsmSaveWorker(pDb, 0); - } - } - } - - /* Clean up the MergeWorker object initialized above. If no error - ** has occurred, invoke the work-hook to inform the application that - ** the database structure has changed. */ - mergeWorkerShutdown(&mergeworker, &rc); - pDb->bIncrMerge = 0; - if( rc==LSM_OK ) sortedInvokeWorkHook(pDb); - -#if LSM_LOG_STRUCTURE - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "work"); -#endif - assertBtreeOk(pDb, &pLevel->lhs); - assertRunInOrder(pDb, &pLevel->lhs); - - /* If bFlush is true and the database is no longer considered "full", - ** break out of the loop even if nRemaining is still greater than - ** zero. The caller has an in-memory tree to flush to disk. */ - if( bFlush && sortedDbIsFull(pDb)==0 ) break; - } - } - - if( pnWrite ) *pnWrite = (nWork - nRemaining); - pWorker->nWrite += (nWork - nRemaining); - -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, rc, "sortedWork(): %d pages", (nWork-nRemaining)); -#endif - return rc; -} - -/* -** The database connection passed as the first argument must be a worker -** connection. This function checks if there exists an "old" in-memory tree -** ready to be flushed to disk. If so, true is returned. Otherwise false. -** -** If an error occurs, *pRc is set to an LSM error code before returning. -** It is assumed that *pRc is set to LSM_OK when this function is called. -*/ -static int sortedTreeHasOld(lsm_db *pDb, int *pRc){ - int rc = LSM_OK; - int bRet = 0; - - assert( pDb->pWorker ); - if( *pRc==LSM_OK ){ - if( rc==LSM_OK - && pDb->treehdr.iOldShmid - && pDb->treehdr.iOldLog!=pDb->pWorker->iLogOff - ){ - bRet = 1; - }else{ - bRet = 0; - } - *pRc = rc; - } - assert( *pRc==LSM_OK || bRet==0 ); - return bRet; -} - -/* -** Create a new free-list only top-level segment. Return LSM_OK if successful -** or an LSM error code if some error occurs. -*/ -static int sortedNewFreelistOnly(lsm_db *pDb){ - return sortedNewToplevel(pDb, TREE_NONE, 0); -} - -int lsmSaveWorker(lsm_db *pDb, int bFlush){ - Snapshot *p = pDb->pWorker; - if( p->freelist.nEntry>pDb->nMaxFreelist ){ - int rc = sortedNewFreelistOnly(pDb); - if( rc!=LSM_OK ) return rc; - } - return lsmCheckpointSaveWorker(pDb, bFlush); -} - -static int doLsmSingleWork( - lsm_db *pDb, - int bShutdown, - int nMerge, /* Minimum segments to merge together */ - int nPage, /* Number of pages to write to disk */ - int *pnWrite, /* OUT: Pages actually written to disk */ - int *pbCkpt /* OUT: True if an auto-checkpoint is req. */ -){ - Snapshot *pWorker; /* Worker snapshot */ - int rc = LSM_OK; /* Return code */ - int bDirty = 0; - int nMax = nPage; /* Maximum pages to write to disk */ - int nRem = nPage; - int bCkpt = 0; - - assert( nPage>0 ); - - /* Open the worker 'transaction'. It will be closed before this function - ** returns. */ - assert( pDb->pWorker==0 ); - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - - /* If this connection is doing auto-checkpoints, set nMax (and nRem) so - ** that this call stops writing when the auto-checkpoint is due. The - ** caller will do the checkpoint, then possibly call this function again. */ - if( bShutdown==0 && pDb->nAutockpt ){ - u32 nSync; - u32 nUnsync; - int nPgsz; - - lsmCheckpointSynced(pDb, 0, 0, &nSync); - nUnsync = lsmCheckpointNWrite(pDb->pShmhdr->aSnap1, 0); - nPgsz = lsmCheckpointPgsz(pDb->pShmhdr->aSnap1); - - nMax = (int)LSM_MIN(nMax, (pDb->nAutockpt/nPgsz) - (int)(nUnsync-nSync)); - if( nMax<nRem ){ - bCkpt = 1; - nRem = LSM_MAX(nMax, 0); - } - } - - /* If there exists in-memory data ready to be flushed to disk, attempt - ** to flush it now. */ - if( pDb->nTransOpen==0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } - if( sortedTreeHasOld(pDb, &rc) ){ - /* sortedDbIsFull() returns non-zero if either (a) there are too many - ** levels in total in the db, or (b) there are too many levels with the - ** the same age in the db. Either way, call sortedWork() to merge - ** existing segments together until this condition is cleared. */ - if( sortedDbIsFull(pDb) ){ - int nPg = 0; - rc = sortedWork(pDb, nRem, nMerge, 1, &nPg); - nRem -= nPg; - assert( rc!=LSM_OK || nRem<=0 || !sortedDbIsFull(pDb) ); - bDirty = 1; - } - - if( rc==LSM_OK && nRem>0 ){ - int nPg = 0; - rc = sortedNewToplevel(pDb, TREE_OLD, &nPg); - nRem -= nPg; - if( rc==LSM_OK ){ - if( pDb->nTransOpen>0 ){ - lsmTreeDiscardOld(pDb); - } - rc = lsmSaveWorker(pDb, 1); - bDirty = 0; - } - } - } - - /* If nPage is still greater than zero, do some merging. */ - if( rc==LSM_OK && nRem>0 && bShutdown==0 ){ - int nPg = 0; - rc = sortedWork(pDb, nRem, nMerge, 0, &nPg); - nRem -= nPg; - if( nPg ) bDirty = 1; - } - - /* If the in-memory part of the free-list is too large, write a new - ** top-level containing just the in-memory free-list entries to disk. */ - if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){ - while( rc==LSM_OK && lsmDatabaseFull(pDb) ){ - int nPg = 0; - rc = sortedWork(pDb, 16, nMerge, 1, &nPg); - nRem -= nPg; - } - if( rc==LSM_OK ){ - rc = sortedNewFreelistOnly(pDb); - } - bDirty = 1; - } - - if( rc==LSM_OK ){ - *pnWrite = (nMax - nRem); - *pbCkpt = (bCkpt && nRem<=0); - if( nMerge==1 && pDb->nAutockpt>0 && *pnWrite>0 - && pWorker->pLevel - && pWorker->pLevel->nRight==0 - && pWorker->pLevel->pNext==0 - ){ - *pbCkpt = 1; - } - } - - if( rc==LSM_OK && bDirty ){ - lsmFinishWork(pDb, 0, &rc); - }else{ - int rcdummy = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcdummy); - *pnWrite = 0; - } - assert( pDb->pWorker==0 ); - return rc; -} - -static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){ - int rc = LSM_OK; /* Return code */ - int nWrite = 0; /* Number of pages written */ - - assert( nMerge>=1 ); - - if( nPage!=0 ){ - int bCkpt = 0; - do { - int nThis = 0; - int nReq = (nPage>=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF); - - bCkpt = 0; - rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt); - nWrite += nThis; - if( rc==LSM_OK && bCkpt ){ - rc = lsm_checkpoint(pDb, 0); - } - }while( rc==LSM_OK && bCkpt && (nWrite<nPage || nPage<0) ); - } - - if( pnWrite ){ - if( rc==LSM_OK ){ - *pnWrite = nWrite; - }else{ - *pnWrite = 0; - } - } - return rc; -} - -/* -** Perform work to merge database segments together. -*/ -int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite){ - int rc; /* Return code */ - int nPgsz; /* Nominal page size in bytes */ - int nPage; /* Equivalent of nKB in pages */ - int nWrite = 0; /* Number of pages written */ - - /* This function may not be called if pDb has an open read or write - ** transaction. Return LSM_MISUSE if an application attempts this. */ - if( pDb->nTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT; - if( nMerge<=0 ) nMerge = pDb->nMerge; - - lsmFsPurgeCache(pDb->pFS); - - /* Convert from KB to pages */ - nPgsz = lsmFsPageSize(pDb->pFS); - if( nKB>=0 ){ - nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz; - }else{ - nPage = -1; - } - - rc = doLsmWork(pDb, nMerge, nPage, &nWrite); - - if( pnWrite ){ - /* Convert back from pages to KB */ - *pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz); - } - return rc; -} - -int lsm_flush(lsm_db *db){ - int rc; - - if( db->nTransOpen>0 || db->pCsr ){ - rc = LSM_MISUSE_BKPT; - }else{ - rc = lsmBeginWriteTrans(db); - if( rc==LSM_OK ){ - lsmFlushTreeToDisk(db); - lsmTreeDiscardOld(db); - lsmTreeMakeOld(db); - lsmTreeDiscardOld(db); - } - - if( rc==LSM_OK ){ - rc = lsmFinishWriteTrans(db, 1); - }else{ - lsmFinishWriteTrans(db, 0); - } - lsmFinishReadTrans(db); - } - - return rc; -} - -/* -** This function is called in auto-work mode to perform merging work on -** the data structure. It performs enough merging work to prevent the -** height of the tree from growing indefinitely assuming that roughly -** nUnit database pages worth of data have been written to the database -** (i.e. the in-memory tree) since the last call. -*/ -int lsmSortedAutoWork( - lsm_db *pDb, /* Database handle */ - int nUnit /* Pages of data written to in-memory tree */ -){ - int rc = LSM_OK; /* Return code */ - int nDepth = 0; /* Current height of tree (longest path) */ - Level *pLevel; /* Used to iterate through levels */ - int bRestore = 0; - - assert( pDb->pWorker==0 ); - assert( pDb->nTransOpen>0 ); - - /* Determine how many units of work to do before returning. One unit of - ** work is achieved by writing one page (~4KB) of merged data. */ - for(pLevel=lsmDbSnapshotLevel(pDb->pClient); pLevel; pLevel=pLevel->pNext){ - /* nDepth += LSM_MAX(1, pLevel->nRight); */ - nDepth += 1; - } - if( lsmTreeHasOld(pDb) ){ - nDepth += 1; - bRestore = 1; - rc = lsmSaveCursors(pDb); - if( rc!=LSM_OK ) return rc; - } - - if( nDepth>0 ){ - int nRemaining; /* Units of work to do before returning */ - - nRemaining = nUnit * nDepth; -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", - nUnit, nDepth, nRemaining); -#endif - assert( nRemaining>=0 ); - rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0); - if( rc==LSM_BUSY ) rc = LSM_OK; - - if( bRestore && pDb->pCsr ){ - lsmMCursorFreeCache(pDb); - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - if( rc==LSM_OK ){ - rc = lsmCheckpointLoad(pDb, 0); - } - if( rc==LSM_OK ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot, &pDb->pClient); - } - if( rc==LSM_OK ){ - rc = lsmRestoreCursors(pDb); - } - } - } - - return rc; -} - -/* -** This function is only called during system shutdown. The contents of -** any in-memory trees present (old or current) are written out to disk. -*/ -int lsmFlushTreeToDisk(lsm_db *pDb){ - int rc; - - rc = lsmBeginWork(pDb); - while( rc==LSM_OK && sortedDbIsFull(pDb) ){ - rc = sortedWork(pDb, 256, pDb->nMerge, 1, 0); - } - - if( rc==LSM_OK ){ - rc = sortedNewToplevel(pDb, TREE_BOTH, 0); - } - - lsmFinishWork(pDb, 1, &rc); - return rc; -} - -/* -** Return a string representation of the segment passed as the only argument. -** Space for the returned string is allocated using lsmMalloc(), and should -** be freed by the caller using lsmFree(). -*/ -static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){ - LsmPgno nSize = pSeg->nSize; - LsmPgno iRoot = pSeg->iRoot; - LsmPgno iFirst = pSeg->iFirst; - LsmPgno iLast = pSeg->iLastPg; - char *z; - - char *z1; - char *z2; - int nPad; - - z1 = lsmMallocPrintf(pEnv, "%d.%d", iFirst, iLast); - if( iRoot ){ - z2 = lsmMallocPrintf(pEnv, "root=%lld", iRoot); - }else{ - z2 = lsmMallocPrintf(pEnv, "size=%lld", nSize); - } - - nPad = nMin - 2 - strlen(z1) - 1 - strlen(z2); - nPad = LSM_MAX(0, nPad); - - if( iRoot ){ - z = lsmMallocPrintf(pEnv, "/%s %*s%s\\", z1, nPad, "", z2); - }else{ - z = lsmMallocPrintf(pEnv, "|%s %*s%s|", z1, nPad, "", z2); - } - lsmFree(pEnv, z1); - lsmFree(pEnv, z2); - - return z; -} - -static int fileToString( - lsm_db *pDb, /* For xMalloc() */ - char *aBuf, - int nBuf, - int nMin, - Segment *pSeg -){ - int i = 0; - if( pSeg ){ - char *zSeg; - - zSeg = segToString(pDb->pEnv, pSeg, nMin); - snprintf(&aBuf[i], nBuf-i, "%s", zSeg); - i += strlen(&aBuf[i]); - lsmFree(pDb->pEnv, zSeg); - -#ifdef LSM_LOG_FREELIST - lsmInfoArrayStructure(pDb, 1, pSeg->iFirst, &zSeg); - snprintf(&aBuf[i], nBuf-1, " (%s)", zSeg); - i += strlen(&aBuf[i]); - lsmFree(pDb->pEnv, zSeg); -#endif - aBuf[nBuf] = 0; - }else{ - aBuf[0] = '\0'; - } - - return i; -} - -void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){ - LsmBlob blob = {0, 0, 0}; /* LsmBlob used for keys */ - LsmString s; - int i; - - int nRec; - LsmPgno iPtr; - int flags; - u8 *aData; - int nData; - - aData = fsPageData(pPg, &nData); - - nRec = pageGetNRec(aData, nData); - iPtr = pageGetPtr(aData, nData); - flags = pageGetFlags(aData, nData); - - lsmStringInit(&s, pDb->pEnv); - lsmStringAppendf(&s,"nCell=%d iPtr=%lld flags=%d {", nRec, iPtr, flags); - if( flags&SEGMENT_BTREE_FLAG ) iPtr = 0; - - for(i=0; i<nRec; i++){ - Page *pRef = 0; /* Pointer to page iRef */ - int iChar; - u8 *aKey; int nKey = 0; /* Key */ - u8 *aVal = 0; int nVal = 0; /* Value */ - int iTopic; - u8 *aCell; - i64 iPgPtr; - int eType; - - aCell = pageGetCell(aData, nData, i); - eType = *aCell++; - assert( (flags & SEGMENT_BTREE_FLAG) || eType!=0 ); - aCell += lsmVarintGet64(aCell, &iPgPtr); - - if( eType==0 ){ - LsmPgno iRef; /* Page number of referenced page */ - aCell += lsmVarintGet64(aCell, &iRef); - lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef); - aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob); - }else{ - aCell += lsmVarintGet32(aCell, &nKey); - if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(0, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob); - aVal = &aKey[nKey]; - iTopic = eType; - } - - lsmStringAppendf(&s, "%s%2X:", (i==0?"":" "), iTopic); - for(iChar=0; iChar<nKey; iChar++){ - lsmStringAppendf(&s, "%c", isalnum(aKey[iChar]) ? aKey[iChar] : '.'); - } - if( nVal>0 && bVals ){ - lsmStringAppendf(&s, "##"); - for(iChar=0; iChar<nVal; iChar++){ - lsmStringAppendf(&s, "%c", isalnum(aVal[iChar]) ? aVal[iChar] : '.'); - } - } - - lsmStringAppendf(&s, " %lld", iPgPtr+iPtr); - lsmFsPageRelease(pRef); - } - lsmStringAppend(&s, "}", 1); - - lsmLogMessage(pDb, LSM_OK, " Page %d: %s", lsmFsPageNumber(pPg), s.z); - lsmStringClear(&s); - - sortedBlobFree(&blob); -} - -static void infoCellDump( - lsm_db *pDb, /* Database handle */ - Segment *pSeg, /* Segment page belongs to */ - int bIndirect, /* True to follow indirect refs */ - Page *pPg, - int iCell, - int *peType, - int *piPgPtr, - u8 **paKey, int *pnKey, - u8 **paVal, int *pnVal, - LsmBlob *pBlob -){ - u8 *aData; int nData; /* Page data */ - u8 *aKey; int nKey = 0; /* Key */ - u8 *aVal = 0; int nVal = 0; /* Value */ - int eType; - int iPgPtr; - Page *pRef = 0; /* Pointer to page iRef */ - u8 *aCell; - - aData = fsPageData(pPg, &nData); - - aCell = pageGetCell(aData, nData, iCell); - eType = *aCell++; - aCell += lsmVarintGet32(aCell, &iPgPtr); - - if( eType==0 ){ - int dummy; - LsmPgno iRef; /* Page number of referenced page */ - aCell += lsmVarintGet64(aCell, &iRef); - if( bIndirect ){ - lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef); - pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob); - aKey = (u8 *)pBlob->pData; - nKey = pBlob->nData; - lsmFsPageRelease(pRef); - }else{ - aKey = (u8 *)"<indirect>"; - nKey = 11; - } - }else{ - aCell += lsmVarintGet32(aCell, &nKey); - if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(pSeg, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, pBlob); - aVal = &aKey[nKey]; - } - - if( peType ) *peType = eType; - if( piPgPtr ) *piPgPtr = iPgPtr; - if( paKey ) *paKey = aKey; - if( paVal ) *paVal = aVal; - if( pnKey ) *pnKey = nKey; - if( pnVal ) *pnVal = nVal; -} - -static int infoAppendBlob(LsmString *pStr, int bHex, u8 *z, int n){ - int iChar; - for(iChar=0; iChar<n; iChar++){ - if( bHex ){ - lsmStringAppendf(pStr, "%02X", z[iChar]); - }else{ - lsmStringAppendf(pStr, "%c", isalnum(z[iChar]) ?z[iChar] : '.'); - } - } - return LSM_OK; -} - -#define INFO_PAGE_DUMP_DATA 0x01 -#define INFO_PAGE_DUMP_VALUES 0x02 -#define INFO_PAGE_DUMP_HEX 0x04 -#define INFO_PAGE_DUMP_INDIRECT 0x08 - -static int infoPageDump( - lsm_db *pDb, /* Database handle */ - LsmPgno iPg, /* Page number of page to dump */ - int flags, - char **pzOut /* OUT: lsmMalloc'd string */ -){ - int rc = LSM_OK; /* Return code */ - Page *pPg = 0; /* Handle for page iPg */ - int i, j; /* Loop counters */ - const int perLine = 16; /* Bytes per line in the raw hex dump */ - Segment *pSeg = 0; - Snapshot *pSnap; - - int bValues = (flags & INFO_PAGE_DUMP_VALUES); - int bHex = (flags & INFO_PAGE_DUMP_HEX); - int bData = (flags & INFO_PAGE_DUMP_DATA); - int bIndirect = (flags & INFO_PAGE_DUMP_INDIRECT); - - *pzOut = 0; - if( iPg==0 ) return LSM_ERROR; - - assert( pDb->pClient || pDb->pWorker ); - pSnap = pDb->pClient; - if( pSnap==0 ) pSnap = pDb->pWorker; - if( pSnap->redirect.n>0 ){ - Level *pLvl; - int bUse = 0; - for(pLvl=pSnap->pLevel; pLvl->pNext; pLvl=pLvl->pNext); - pSeg = (pLvl->nRight==0 ? &pLvl->lhs : &pLvl->aRhs[pLvl->nRight-1]); - rc = lsmFsSegmentContainsPg(pDb->pFS, pSeg, iPg, &bUse); - if( bUse==0 ){ - pSeg = 0; - } - } - - /* iPg is a real page number (not subject to redirection). So it is safe - ** to pass a NULL in place of the segment pointer as the second argument - ** to lsmFsDbPageGet() here. */ - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg); - } - - if( rc==LSM_OK ){ - LsmBlob blob = {0, 0, 0, 0}; - int nKeyWidth = 0; - LsmString str; - int nRec; - LsmPgno iPtr; - int flags2; - int iCell; - u8 *aData; int nData; /* Page data and size thereof */ - - aData = fsPageData(pPg, &nData); - nRec = pageGetNRec(aData, nData); - iPtr = pageGetPtr(aData, nData); - flags2 = pageGetFlags(aData, nData); - - lsmStringInit(&str, pDb->pEnv); - lsmStringAppendf(&str, "Page : %lld (%d bytes)\n", iPg, nData); - lsmStringAppendf(&str, "nRec : %d\n", nRec); - lsmStringAppendf(&str, "iPtr : %lld\n", iPtr); - lsmStringAppendf(&str, "flags: %04x\n", flags2); - lsmStringAppendf(&str, "\n"); - - for(iCell=0; iCell<nRec; iCell++){ - int nKey; - infoCellDump( - pDb, pSeg, bIndirect, pPg, iCell, 0, 0, 0, &nKey, 0, 0, &blob - ); - if( nKey>nKeyWidth ) nKeyWidth = nKey; - } - if( bHex ) nKeyWidth = nKeyWidth * 2; - - for(iCell=0; iCell<nRec; iCell++){ - u8 *aKey; int nKey = 0; /* Key */ - u8 *aVal; int nVal = 0; /* Value */ - int iPgPtr; - int eType; - LsmPgno iAbsPtr; - char zFlags[8]; - - infoCellDump(pDb, pSeg, bIndirect, pPg, iCell, &eType, &iPgPtr, - &aKey, &nKey, &aVal, &nVal, &blob - ); - iAbsPtr = iPgPtr + ((flags2 & SEGMENT_BTREE_FLAG) ? 0 : iPtr); - - lsmFlagsToString(eType, zFlags); - lsmStringAppendf(&str, "%s %d (%s) ", - zFlags, iAbsPtr, (rtTopic(eType) ? "sys" : "usr") - ); - infoAppendBlob(&str, bHex, aKey, nKey); - if( nVal>0 && bValues ){ - lsmStringAppendf(&str, "%*s", nKeyWidth - (nKey*(1+bHex)), ""); - lsmStringAppendf(&str, " "); - infoAppendBlob(&str, bHex, aVal, nVal); - } - if( rtTopic(eType) ){ - int iBlk = (int)~lsmGetU32(aKey); - lsmStringAppendf(&str, " (block=%d", iBlk); - if( nVal>0 ){ - i64 iSnap = lsmGetU64(aVal); - lsmStringAppendf(&str, " snapshot=%lld", iSnap); - } - lsmStringAppendf(&str, ")"); - } - lsmStringAppendf(&str, "\n"); - } - - if( bData ){ - lsmStringAppendf(&str, "\n-------------------" - "-------------------------------------------------------------\n"); - lsmStringAppendf(&str, "Page %d\n", - iPg, (iPg-1)*nData, iPg*nData - 1); - for(i=0; i<nData; i += perLine){ - lsmStringAppendf(&str, "%04x: ", i); - for(j=0; j<perLine; j++){ - if( i+j>nData ){ - lsmStringAppendf(&str, " "); - }else{ - lsmStringAppendf(&str, "%02x ", aData[i+j]); - } - } - lsmStringAppendf(&str, " "); - for(j=0; j<perLine; j++){ - if( i+j>nData ){ - lsmStringAppendf(&str, " "); - }else{ - lsmStringAppendf(&str,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); - } - } - lsmStringAppendf(&str,"\n"); - } - } - - *pzOut = str.z; - sortedBlobFree(&blob); - lsmFsPageRelease(pPg); - } - - return rc; -} - -int lsmInfoPageDump( - lsm_db *pDb, /* Database handle */ - LsmPgno iPg, /* Page number of page to dump */ - int bHex, /* True to output key/value in hex form */ - char **pzOut /* OUT: lsmMalloc'd string */ -){ - int flags = INFO_PAGE_DUMP_DATA | INFO_PAGE_DUMP_VALUES; - if( bHex ) flags |= INFO_PAGE_DUMP_HEX; - return infoPageDump(pDb, iPg, flags, pzOut); -} - -void sortedDumpSegment(lsm_db *pDb, Segment *pRun, int bVals){ - assert( pDb->xLog ); - if( pRun && pRun->iFirst ){ - int flags = (bVals ? INFO_PAGE_DUMP_VALUES : 0); - char *zSeg; - Page *pPg; - - zSeg = segToString(pDb->pEnv, pRun, 0); - lsmLogMessage(pDb, LSM_OK, "Segment: %s", zSeg); - lsmFree(pDb->pEnv, zSeg); - - lsmFsDbPageGet(pDb->pFS, pRun, pRun->iFirst, &pPg); - while( pPg ){ - Page *pNext; - char *z = 0; - infoPageDump(pDb, lsmFsPageNumber(pPg), flags, &z); - lsmLogMessage(pDb, LSM_OK, "%s", z); - lsmFree(pDb->pEnv, z); -#if 0 - sortedDumpPage(pDb, pRun, pPg, bVals); -#endif - lsmFsDbPageNext(pRun, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - } -} - -/* -** Invoke the log callback zero or more times with messages that describe -** the current database structure. -*/ -void lsmSortedDumpStructure( - lsm_db *pDb, /* Database handle (used for xLog callback) */ - Snapshot *pSnap, /* Snapshot to dump */ - int bKeys, /* Output the keys from each segment */ - int bVals, /* Output the values from each segment */ - const char *zWhy /* Caption to print near top of dump */ -){ - Snapshot *pDump = pSnap; - Level *pTopLevel; - char *zFree = 0; - - assert( pSnap ); - pTopLevel = lsmDbSnapshotLevel(pDump); - if( pDb->xLog && pTopLevel ){ - static int nCall = 0; - Level *pLevel; - int iLevel = 0; - - nCall++; - lsmLogMessage(pDb, LSM_OK, "Database structure %d (%s)", nCall, zWhy); - -#if 0 - if( nCall==1031 || nCall==1032 ) bKeys=1; -#endif - - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - char zLeft[1024]; - char zRight[1024]; - int i = 0; - - Segment *aLeft[24]; - Segment *aRight[24]; - - int nLeft = 0; - int nRight = 0; - - Segment *pSeg = &pLevel->lhs; - aLeft[nLeft++] = pSeg; - - for(i=0; i<pLevel->nRight; i++){ - aRight[nRight++] = &pLevel->aRhs[i]; - } - -#ifdef LSM_LOG_FREELIST - if( nRight ){ - memmove(&aRight[1], aRight, sizeof(aRight[0])*nRight); - aRight[0] = 0; - nRight++; - } -#endif - - for(i=0; i<nLeft || i<nRight; i++){ - int iPad = 0; - char zLevel[32]; - zLeft[0] = '\0'; - zRight[0] = '\0'; - - if( i<nLeft ){ - fileToString(pDb, zLeft, sizeof(zLeft), 24, aLeft[i]); - } - if( i<nRight ){ - fileToString(pDb, zRight, sizeof(zRight), 24, aRight[i]); - } - - if( i==0 ){ - snprintf(zLevel, sizeof(zLevel), "L%d: (age=%d) (flags=%.4x)", - iLevel, (int)pLevel->iAge, (int)pLevel->flags - ); - }else{ - zLevel[0] = '\0'; - } - - if( nRight==0 ){ - iPad = 10; - } - - lsmLogMessage(pDb, LSM_OK, "% 25s % *s% -35s %s", - zLevel, iPad, "", zLeft, zRight - ); - } - - iLevel++; - } - - if( bKeys ){ - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - int i; - sortedDumpSegment(pDb, &pLevel->lhs, bVals); - for(i=0; i<pLevel->nRight; i++){ - sortedDumpSegment(pDb, &pLevel->aRhs[i], bVals); - } - } - } - } - - lsmInfoFreelist(pDb, &zFree); - lsmLogMessage(pDb, LSM_OK, "Freelist: %s", zFree); - lsmFree(pDb->pEnv, zFree); - - assert( lsmFsIntegrityCheck(pDb) ); -} - -void lsmSortedFreeLevel(lsm_env *pEnv, Level *pLevel){ - Level *pNext; - Level *p; - - for(p=pLevel; p; p=pNext){ - pNext = p->pNext; - sortedFreeLevel(pEnv, p); - } -} - -void lsmSortedSaveTreeCursors(lsm_db *pDb){ - MultiCursor *pCsr; - for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ - lsmTreeCursorSave(pCsr->apTreeCsr[0]); - lsmTreeCursorSave(pCsr->apTreeCsr[1]); - } -} - -void lsmSortedExpandBtreePage(Page *pPg, int nOrig){ - u8 *aData; - int nData; - int nEntry; - int iHdr; - - aData = lsmFsPageData(pPg, &nData); - nEntry = pageGetNRec(aData, nOrig); - iHdr = SEGMENT_EOF(nOrig, nEntry); - memmove(&aData[iHdr + (nData-nOrig)], &aData[iHdr], nOrig-iHdr); -} - -#ifdef LSM_DEBUG_EXPENSIVE -static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){ - Page *pPg = 0; - LsmBlob blob1 = {0, 0, 0, 0}; - LsmBlob blob2 = {0, 0, 0, 0}; - - lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg); - while( pPg ){ - u8 *aData; int nData; - Page *pNext; - - aData = lsmFsPageData(pPg, &nData); - if( 0==(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ){ - int i; - int nRec = pageGetNRec(aData, nData); - for(i=0; i<nRec; i++){ - int iTopic1, iTopic2; - pageGetKeyCopy(pDb->pEnv, pSeg, pPg, i, &iTopic1, &blob1); - - if( i==0 && blob2.nData ){ - assert( sortedKeyCompare( - pDb->xCmp, iTopic2, blob2.pData, blob2.nData, - iTopic1, blob1.pData, blob1.nData - )<0 ); - } - - if( i<(nRec-1) ){ - pageGetKeyCopy(pDb->pEnv, pSeg, pPg, i+1, &iTopic2, &blob2); - assert( sortedKeyCompare( - pDb->xCmp, iTopic1, blob1.pData, blob1.nData, - iTopic2, blob2.pData, blob2.nData - )<0 ); - } - } - } - - lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - - sortedBlobFree(&blob1); - sortedBlobFree(&blob2); -} -#endif - -#ifdef LSM_DEBUG_EXPENSIVE -/* -** This function is only included in the build if LSM_DEBUG_EXPENSIVE is -** defined. Its only purpose is to evaluate various assert() statements to -** verify that the database is well formed in certain respects. -** -** More specifically, it checks that the array pOne contains the required -** pointers to pTwo. Array pTwo must be a main array. pOne may be either a -** separators array or another main array. If pOne does not contain the -** correct set of pointers, an assert() statement fails. -*/ -static int assertPointersOk( - lsm_db *pDb, /* Database handle */ - Segment *pOne, /* Segment containing pointers */ - Segment *pTwo, /* Segment containing pointer targets */ - int bRhs /* True if pTwo may have been Gobble()d */ -){ - int rc = LSM_OK; /* Error code */ - SegmentPtr ptr1; /* Iterates through pOne */ - SegmentPtr ptr2; /* Iterates through pTwo */ - LsmPgno iPrev; - - assert( pOne && pTwo ); - - memset(&ptr1, 0, sizeof(ptr1)); - memset(&ptr2, 0, sizeof(ptr1)); - ptr1.pSeg = pOne; - ptr2.pSeg = pTwo; - segmentPtrEndPage(pDb->pFS, &ptr1, 0, &rc); - segmentPtrEndPage(pDb->pFS, &ptr2, 0, &rc); - - /* Check that the footer pointer of the first page of pOne points to - ** the first page of pTwo. */ - iPrev = pTwo->iFirst; - if( ptr1.iPtr!=iPrev && !bRhs ){ - assert( 0 ); - } - - if( rc==LSM_OK && ptr1.nCell>0 ){ - rc = segmentPtrLoadCell(&ptr1, 0); - } - - while( rc==LSM_OK && ptr2.pPg ){ - LsmPgno iThis; - - /* Advance to the next page of segment pTwo that contains at least - ** one cell. Break out of the loop if the iterator reaches EOF. */ - do{ - rc = segmentPtrNextPage(&ptr2, 1); - assert( rc==LSM_OK ); - }while( rc==LSM_OK && ptr2.pPg && ptr2.nCell==0 ); - if( rc!=LSM_OK || ptr2.pPg==0 ) break; - iThis = lsmFsPageNumber(ptr2.pPg); - - if( (ptr2.flags & (PGFTR_SKIP_THIS_FLAG|SEGMENT_BTREE_FLAG))==0 ){ - - /* Load the first cell in the array pTwo page. */ - rc = segmentPtrLoadCell(&ptr2, 0); - - /* Iterate forwards through pOne, searching for a key that matches the - ** key ptr2.pKey/nKey. This key should have a pointer to the page that - ** ptr2 currently points to. */ - while( rc==LSM_OK ){ - int res = rtTopic(ptr1.eType) - rtTopic(ptr2.eType); - if( res==0 ){ - res = pDb->xCmp(ptr1.pKey, ptr1.nKey, ptr2.pKey, ptr2.nKey); - } - - if( res<0 ){ - assert( bRhs || ptr1.iPtr+ptr1.iPgPtr==iPrev ); - }else if( res>0 ){ - assert( 0 ); - }else{ - assert( ptr1.iPtr+ptr1.iPgPtr==iThis ); - iPrev = iThis; - break; - } - - rc = segmentPtrAdvance(0, &ptr1, 0); - if( ptr1.pPg==0 ){ - assert( 0 ); - } - } - } - } - - segmentPtrReset(&ptr1, 0); - segmentPtrReset(&ptr2, 0); - return LSM_OK; -} - -/* -** This function is only included in the build if LSM_DEBUG_EXPENSIVE is -** defined. Its only purpose is to evaluate various assert() statements to -** verify that the database is well formed in certain respects. -** -** More specifically, it checks that the b-tree embedded in array pRun -** contains the correct keys. If not, an assert() fails. -*/ -static int assertBtreeOk( - lsm_db *pDb, - Segment *pSeg -){ - int rc = LSM_OK; /* Return code */ - if( pSeg->iRoot ){ - LsmBlob blob = {0, 0, 0}; /* Buffer used to cache overflow keys */ - FileSystem *pFS = pDb->pFS; /* File system to read from */ - Page *pPg = 0; /* Main run page */ - BtreeCursor *pCsr = 0; /* Btree cursor */ - - rc = btreeCursorNew(pDb, pSeg, &pCsr); - if( rc==LSM_OK ){ - rc = btreeCursorFirst(pCsr); - } - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pPg); - } - - while( rc==LSM_OK ){ - Page *pNext; - u8 *aData; - int nData; - int flags; - - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - if( pPg==0 ) break; - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( rc==LSM_OK - && 0==((SEGMENT_BTREE_FLAG|PGFTR_SKIP_THIS_FLAG) & flags) - && 0!=pageGetNRec(aData, nData) - ){ - u8 *pKey; - int nKey; - int iTopic; - pKey = pageGetKey(pSeg, pPg, 0, &iTopic, &nKey, &blob); - assert( nKey==pCsr->nKey && 0==memcmp(pKey, pCsr->pKey, nKey) ); - assert( lsmFsPageNumber(pPg)==pCsr->iPtr ); - rc = btreeCursorNext(pCsr); - } - } - assert( rc!=LSM_OK || pCsr->pKey==0 ); - - if( pPg ) lsmFsPageRelease(pPg); - - btreeCursorFree(pCsr); - sortedBlobFree(&blob); - } - - return rc; -} -#endif /* ifdef LSM_DEBUG_EXPENSIVE */ diff --git a/ext/lsm1/lsm_str.c b/ext/lsm1/lsm_str.c deleted file mode 100644 index 9b1b63cee..000000000 --- a/ext/lsm1/lsm_str.c +++ /dev/null @@ -1,148 +0,0 @@ -/* -** 2012-04-27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Dynamic string functions. -*/ -#include "lsmInt.h" - -/* -** Turn bulk and uninitialized memory into an LsmString object -*/ -void lsmStringInit(LsmString *pStr, lsm_env *pEnv){ - memset(pStr, 0, sizeof(pStr[0])); - pStr->pEnv = pEnv; -} - -/* -** Increase the memory allocated for holding the string. Realloc as needed. -** -** If a memory allocation error occurs, set pStr->n to -1 and free the existing -** allocation. If a prior memory allocation has occurred, this routine is a -** no-op. -*/ -int lsmStringExtend(LsmString *pStr, int nNew){ - assert( nNew>0 ); - if( pStr->n<0 ) return LSM_NOMEM; - if( pStr->n + nNew >= pStr->nAlloc ){ - int nAlloc = pStr->n + nNew + 100; - char *zNew = lsmRealloc(pStr->pEnv, pStr->z, nAlloc); - if( zNew==0 ){ - lsmFree(pStr->pEnv, pStr->z); - nAlloc = 0; - pStr->n = -1; - } - pStr->nAlloc = nAlloc; - pStr->z = zNew; - } - return (pStr->z ? LSM_OK : LSM_NOMEM_BKPT); -} - -/* -** Clear an LsmString object, releasing any allocated memory that it holds. -** This also clears the error indication (if any). -*/ -void lsmStringClear(LsmString *pStr){ - lsmFree(pStr->pEnv, pStr->z); - lsmStringInit(pStr, pStr->pEnv); -} - -/* -** Append N bytes of text to the end of an LsmString object. If -** N is negative, append the entire string. -** -** If the string is in an error state, this routine is a no-op. -*/ -int lsmStringAppend(LsmString *pStr, const char *z, int N){ - int rc; - if( N<0 ) N = (int)strlen(z); - rc = lsmStringExtend(pStr, N+1); - if( pStr->nAlloc ){ - memcpy(pStr->z+pStr->n, z, N+1); - pStr->n += N; - } - return rc; -} - -int lsmStringBinAppend(LsmString *pStr, const u8 *a, int n){ - int rc; - rc = lsmStringExtend(pStr, n); - if( pStr->nAlloc ){ - memcpy(pStr->z+pStr->n, a, n); - pStr->n += n; - } - return rc; -} - -/* -** Append printf-formatted content to an LsmString. -*/ -void lsmStringVAppendf( - LsmString *pStr, - const char *zFormat, - va_list ap1, - va_list ap2 -){ -#if (!defined(__STDC_VERSION__) || (__STDC_VERSION__<199901L)) && \ - !defined(__APPLE__) - extern int vsnprintf(char *str, size_t size, const char *format, va_list ap) - /* Compatibility crutch for C89 compilation mode. sqlite3_vsnprintf() - does not work identically and causes test failures if used here. - For the time being we are assuming that the target has vsnprintf(), - but that is not guaranteed to be the case for pure C89 platforms. - */; -#endif - int nWrite; - int nAvail; - - nAvail = pStr->nAlloc - pStr->n; - nWrite = vsnprintf(pStr->z + pStr->n, nAvail, zFormat, ap1); - - if( nWrite>=nAvail ){ - lsmStringExtend(pStr, nWrite+1); - if( pStr->nAlloc==0 ) return; - nWrite = vsnprintf(pStr->z + pStr->n, nWrite+1, zFormat, ap2); - } - - pStr->n += nWrite; - pStr->z[pStr->n] = 0; -} - -void lsmStringAppendf(LsmString *pStr, const char *zFormat, ...){ - va_list ap, ap2; - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(pStr, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); -} - -int lsmStrlen(const char *zName){ - int nRet = 0; - while( zName[nRet] ) nRet++; - return nRet; -} - -/* -** Write into memory obtained from lsm_malloc(). -*/ -char *lsmMallocPrintf(lsm_env *pEnv, const char *zFormat, ...){ - LsmString s; - va_list ap, ap2; - lsmStringInit(&s, pEnv); - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(&s, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); - if( s.n<0 ) return 0; - return (char *)lsmReallocOrFree(pEnv, s.z, s.n+1); -} diff --git a/ext/lsm1/lsm_tree.c b/ext/lsm1/lsm_tree.c deleted file mode 100644 index 1a199fc1c..000000000 --- a/ext/lsm1/lsm_tree.c +++ /dev/null @@ -1,2465 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the implementation of an in-memory tree structure. -** -** Technically the tree is a B-tree of order 4 (in the Knuth sense - each -** node may have up to 4 children). Keys are stored within B-tree nodes by -** reference. This may be slightly slower than a conventional red-black -** tree, but it is simpler. It is also an easier structure to modify to -** create a version that supports nested transaction rollback. -** -** This tree does not currently support a delete operation. One is not -** required. When LSM deletes a key from a database, it inserts a DELETE -** marker into the data structure. As a result, although the value associated -** with a key stored in the in-memory tree structure may be modified, no -** keys are ever removed. -*/ - -/* -** MVCC NOTES -** -** The in-memory tree structure supports SQLite-style MVCC. This means -** that while one client is writing to the tree structure, other clients -** may still be querying an older snapshot of the tree. -** -** One way to implement this is to use an append-only b-tree. In this -** case instead of modifying nodes in-place, a copy of the node is made -** and the required modifications made to the copy. The parent of the -** node is then modified (to update the pointer so that it points to -** the new copy), which causes a copy of the parent to be made, and so on. -** This means that each time the tree is written to a new root node is -** created. A snapshot is identified by the root node that it uses. -** -** The problem with the above is that each time the tree is written to, -** a copy of the node structure modified and all of its ancestor nodes -** is made. This may prove excessive with large tree structures. -** -** To reduce this overhead, the data structure used for a tree node is -** designed so that it may be edited in place exactly once without -** affecting existing users. In other words, the node structure is capable -** of storing two separate versions of the node at the same time. -** When a node is to be edited, if the node structure already contains -** two versions, a copy is made as in the append-only approach. Or, if -** it only contains a single version, it is edited in place. -** -** This reduces the overhead so that, roughly, one new node structure -** must be allocated for each write (on top of those allocations that -** would have been required by a non-MVCC tree). Logic: Assume that at -** any time, 50% of nodes in the tree already contain 2 versions. When -** a new entry is written to a node, there is a 50% chance that a copy -** of the node will be required. And a 25% chance that a copy of its -** parent is required. And so on. -** -** ROLLBACK -** -** The in-memory tree also supports transaction and sub-transaction -** rollback. In order to rollback to point in time X, the following is -** necessary: -** -** 1. All memory allocated since X must be freed, and -** 2. All "v2" data adding to nodes that existed at X should be zeroed. -** 3. The root node must be restored to its X value. -** -** The Mempool object used to allocate memory for the tree supports -** operation (1) - see the lsmPoolMark() and lsmPoolRevert() functions. -** -** To support (2), all nodes that have v2 data are part of a singly linked -** list, sorted by the age of the v2 data (nodes that have had data added -** most recently are at the end of the list). So to zero all v2 data added -** since X, the linked list is traversed from the first node added following -** X onwards. -** -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -#include <string.h> - -#define MAX_DEPTH 32 - -typedef struct TreeKey TreeKey; -typedef struct TreeNode TreeNode; -typedef struct TreeLeaf TreeLeaf; -typedef struct NodeVersion NodeVersion; - -struct TreeOld { - u32 iShmid; /* Last shared-memory chunk in use by old */ - u32 iRoot; /* Offset of root node in shm file */ - u32 nHeight; /* Height of tree structure */ -}; - -#if 0 -/* -** assert() that a TreeKey.flags value is sane. Usage: -** -** assert( lsmAssertFlagsOk(pTreeKey->flags) ); -*/ -static int lsmAssertFlagsOk(u8 keyflags){ - /* At least one flag must be set. Otherwise, what is this key doing? */ - assert( keyflags!=0 ); - - /* The POINT_DELETE and INSERT flags cannot both be set. */ - assert( (keyflags & LSM_POINT_DELETE)==0 || (keyflags & LSM_INSERT)==0 ); - - /* If both the START_DELETE and END_DELETE flags are set, then the INSERT - ** flag must also be set. In other words - the three DELETE flags cannot - ** all be set */ - assert( (keyflags & LSM_END_DELETE)==0 - || (keyflags & LSM_START_DELETE)==0 - || (keyflags & LSM_POINT_DELETE)==0 - ); - - return 1; -} -#endif -static int assert_delete_ranges_match(lsm_db *); -static int treeCountEntries(lsm_db *db); - -/* -** Container for a key-value pair. Within the *-shm file, each key/value -** pair is stored in a single allocation (which may not actually be -** contiguous in memory). Layout is the TreeKey structure, followed by -** the nKey bytes of key blob, followed by the nValue bytes of value blob -** (if nValue is non-negative). -*/ -struct TreeKey { - int nKey; /* Size of pKey in bytes */ - int nValue; /* Size of pValue. Or negative. */ - u8 flags; /* Various LSM_XXX flags */ -}; - -#define TKV_KEY(p) ((void *)&(p)[1]) -#define TKV_VAL(p) ((void *)(((u8 *)&(p)[1]) + (p)->nKey)) - -/* -** A single tree node. A node structure may contain up to 3 key/value -** pairs. Internal (non-leaf) nodes have up to 4 children. -** -** TODO: Update the format of this to be more compact. Get it working -** first though... -*/ -struct TreeNode { - u32 aiKeyPtr[3]; /* Array of pointers to TreeKey objects */ - - /* The following fields are present for interior nodes only, not leaves. */ - u32 aiChildPtr[4]; /* Array of pointers to child nodes */ - - /* The extra child pointer slot. */ - u32 iV2; /* Transaction number of v2 */ - u8 iV2Child; /* apChild[] entry replaced by pV2Ptr */ - u32 iV2Ptr; /* Substitute pointer */ -}; - -struct TreeLeaf { - u32 aiKeyPtr[3]; /* Array of pointers to TreeKey objects */ -}; - -typedef struct TreeBlob TreeBlob; -struct TreeBlob { - int n; - u8 *a; -}; - -/* -** Cursor for searching a tree structure. -** -** If a cursor does not point to any element (a.k.a. EOF), then the -** TreeCursor.iNode variable is set to a negative value. Otherwise, the -** cursor currently points to key aiCell[iNode] on node apTreeNode[iNode]. -** -** Entries in the apTreeNode[] and aiCell[] arrays contain the node and -** index of the TreeNode.apChild[] pointer followed to descend to the -** current element. Hence apTreeNode[0] always contains the root node of -** the tree. -*/ -struct TreeCursor { - lsm_db *pDb; /* Database handle for this cursor */ - TreeRoot *pRoot; /* Root node and height of tree to access */ - int iNode; /* Cursor points at apTreeNode[iNode] */ - TreeNode *apTreeNode[MAX_DEPTH];/* Current position in tree */ - u8 aiCell[MAX_DEPTH]; /* Current position in tree */ - TreeKey *pSave; /* Saved key */ - TreeBlob blob; /* Dynamic storage for a key */ -}; - -/* -** A value guaranteed to be larger than the largest possible transaction -** id (TreeHeader.iTransId). -*/ -#define WORKING_VERSION (1<<30) - -static int tblobGrow(lsm_db *pDb, TreeBlob *p, int n, int *pRc){ - if( n>p->n ){ - lsmFree(pDb->pEnv, p->a); - p->a = lsmMallocRc(pDb->pEnv, n, pRc); - p->n = n; - } - return (p->a==0); -} -static void tblobFree(lsm_db *pDb, TreeBlob *p){ - lsmFree(pDb->pEnv, p->a); -} - - -/*********************************************************************** -** Start of IntArray methods. */ -/* -** Append value iVal to the contents of IntArray *p. Return LSM_OK if -** successful, or LSM_NOMEM if an OOM condition is encountered. -*/ -static int intArrayAppend(lsm_env *pEnv, IntArray *p, u32 iVal){ - assert( p->nArray<=p->nAlloc ); - if( p->nArray>=p->nAlloc ){ - u32 *aNew; - int nNew = p->nArray ? p->nArray*2 : 128; - aNew = lsmRealloc(pEnv, p->aArray, nNew*sizeof(u32)); - if( !aNew ) return LSM_NOMEM_BKPT; - p->aArray = aNew; - p->nAlloc = nNew; - } - - p->aArray[p->nArray++] = iVal; - return LSM_OK; -} - -/* -** Zero the IntArray object. -*/ -static void intArrayFree(lsm_env *pEnv, IntArray *p){ - p->nArray = 0; -} - -/* -** Return the number of entries currently in the int-array object. -*/ -static int intArraySize(IntArray *p){ - return p->nArray; -} - -/* -** Return a copy of the iIdx'th entry in the int-array. -*/ -static u32 intArrayEntry(IntArray *p, int iIdx){ - return p->aArray[iIdx]; -} - -/* -** Truncate the int-array so that all but the first nVal values are -** discarded. -*/ -static void intArrayTruncate(IntArray *p, int nVal){ - p->nArray = nVal; -} -/* End of IntArray methods. -***********************************************************************/ - -static int treeKeycmp(void *p1, int n1, void *p2, int n2){ - int res; - res = memcmp(p1, p2, LSM_MIN(n1, n2)); - if( res==0 ) res = (n1-n2); - return res; -} - -/* -** The pointer passed as the first argument points to an interior node, -** not a leaf. This function returns the offset of the iCell'th child -** sub-tree of the node. -*/ -static u32 getChildPtr(TreeNode *p, int iVersion, int iCell){ - assert( iVersion>=0 ); - assert( iCell>=0 && iCell<=array_size(p->aiChildPtr) ); - if( p->iV2 && p->iV2<=(u32)iVersion && iCell==p->iV2Child ) return p->iV2Ptr; - return p->aiChildPtr[iCell]; -} - -/* -** Given an offset within the *-shm file, return the associated chunk number. -*/ -static int treeOffsetToChunk(u32 iOff){ - assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); - return (int)(iOff>>15); -} - -#define treeShmptrUnsafe(pDb, iPtr) \ -(&((u8*)((pDb)->apShm[(iPtr)>>15]))[(iPtr) & (LSM_SHM_CHUNK_SIZE-1)]) - -/* -** Return a pointer to the mapped memory location associated with *-shm -** file offset iPtr. -*/ -static void *treeShmptr(lsm_db *pDb, u32 iPtr){ - - assert( (iPtr>>15)<(u32)pDb->nShm ); - assert( pDb->apShm[iPtr>>15] ); - - return iPtr ? treeShmptrUnsafe(pDb, iPtr) : 0; -} - -static ShmChunk * treeShmChunk(lsm_db *pDb, int iChunk){ - return (ShmChunk *)(pDb->apShm[iChunk]); -} - -static ShmChunk * treeShmChunkRc(lsm_db *pDb, int iChunk, int *pRc){ - assert( *pRc==LSM_OK ); - if( iChunk<pDb->nShm || LSM_OK==(*pRc = lsmShmCacheChunks(pDb, iChunk+1)) ){ - return (ShmChunk *)(pDb->apShm[iChunk]); - } - return 0; -} - - -#ifndef NDEBUG -static void assertIsWorkingChild( - lsm_db *db, - TreeNode *pNode, - TreeNode *pParent, - int iCell -){ - TreeNode *p; - u32 iPtr = getChildPtr(pParent, WORKING_VERSION, iCell); - p = treeShmptr(db, iPtr); - assert( p==pNode ); -} -#else -# define assertIsWorkingChild(w,x,y,z) -#endif - -/* Values for the third argument to treeShmkey(). */ -#define TKV_LOADKEY 1 -#define TKV_LOADVAL 2 - -static TreeKey *treeShmkey( - lsm_db *pDb, /* Database handle */ - u32 iPtr, /* Shmptr to TreeKey struct */ - int eLoad, /* Either zero or a TREEKEY_LOADXXX value */ - TreeBlob *pBlob, /* Used if dynamic memory is required */ - int *pRc /* IN/OUT: Error code */ -){ - TreeKey *pRet; - - assert( eLoad==TKV_LOADKEY || eLoad==TKV_LOADVAL ); - pRet = (TreeKey *)treeShmptr(pDb, iPtr); - if( pRet ){ - int nReq; /* Bytes of space required at pRet */ - int nAvail; /* Bytes of space available at pRet */ - - nReq = sizeof(TreeKey) + pRet->nKey; - if( eLoad==TKV_LOADVAL && pRet->nValue>0 ){ - nReq += pRet->nValue; - } - assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); - nAvail = LSM_SHM_CHUNK_SIZE - (iPtr & (LSM_SHM_CHUNK_SIZE-1)); - - if( nAvail<nReq ){ - if( tblobGrow(pDb, pBlob, nReq, pRc)==0 ){ - int nLoad = 0; - while( *pRc==LSM_OK ){ - ShmChunk *pChunk; - void *p = treeShmptr(pDb, iPtr); - int n = LSM_MIN(nAvail, nReq-nLoad); - - memcpy(&pBlob->a[nLoad], p, n); - nLoad += n; - if( nLoad==nReq ) break; - - pChunk = treeShmChunk(pDb, treeOffsetToChunk(iPtr)); - assert( pChunk ); - iPtr = (pChunk->iNext * LSM_SHM_CHUNK_SIZE) + LSM_SHM_CHUNK_HDR; - nAvail = LSM_SHM_CHUNK_SIZE - LSM_SHM_CHUNK_HDR; - } - } - pRet = (TreeKey *)(pBlob->a); - } - } - - return pRet; -} - -#if defined(LSM_DEBUG) && defined(LSM_EXPENSIVE_ASSERT) -void assert_leaf_looks_ok(TreeNode *pNode){ - assert( pNode->apKey[1] ); -} - -void assert_node_looks_ok(TreeNode *pNode, int nHeight){ - if( pNode ){ - assert( pNode->apKey[1] ); - if( nHeight>1 ){ - int i; - assert( getChildPtr(pNode, WORKING_VERSION, 1) ); - assert( getChildPtr(pNode, WORKING_VERSION, 2) ); - for(i=0; i<4; i++){ - assert_node_looks_ok(getChildPtr(pNode, WORKING_VERSION, i), nHeight-1); - } - } - } -} - -/* -** Run various assert() statements to check that the working-version of the -** tree is correct in the following respects: -** -** * todo... -*/ -void assert_tree_looks_ok(int rc, Tree *pTree){ -} -#else -# define assert_tree_looks_ok(x,y) -#endif - -void lsmFlagsToString(int flags, char *zFlags){ - - zFlags[0] = (flags & LSM_END_DELETE) ? ']' : '.'; - - /* Only one of LSM_POINT_DELETE, LSM_INSERT and LSM_SEPARATOR should ever - ** be set. If this is not true, write a '?' to the output. */ - switch( flags & (LSM_POINT_DELETE|LSM_INSERT|LSM_SEPARATOR) ){ - case 0: zFlags[1] = '.'; break; - case LSM_POINT_DELETE: zFlags[1] = '-'; break; - case LSM_INSERT: zFlags[1] = '+'; break; - case LSM_SEPARATOR: zFlags[1] = '^'; break; - default: zFlags[1] = '?'; break; - } - - zFlags[2] = (flags & LSM_SYSTEMKEY) ? '*' : '.'; - zFlags[3] = (flags & LSM_START_DELETE) ? '[' : '.'; - zFlags[4] = '\0'; -} - -#ifdef LSM_DEBUG - -/* -** Pointer pBlob points to a buffer containing a blob of binary data -** nBlob bytes long. Append the contents of this blob to *pStr, with -** each octet represented by a 2-digit hexadecimal number. For example, -** if the input blob is three bytes in size and contains {0x01, 0x44, 0xFF}, -** then "0144ff" is appended to *pStr. -*/ -static void lsmAppendStrBlob(LsmString *pStr, void *pBlob, int nBlob){ - int i; - lsmStringExtend(pStr, nBlob*2); - if( pStr->nAlloc==0 ) return; - for(i=0; i<nBlob; i++){ - u8 c = ((u8*)pBlob)[i]; - if( c>='a' && c<='z' ){ - pStr->z[pStr->n++] = c; - }else if( c!=0 || nBlob==1 || i!=(nBlob-1) ){ - pStr->z[pStr->n++] = "0123456789abcdef"[(c>>4)&0xf]; - pStr->z[pStr->n++] = "0123456789abcdef"[c&0xf]; - } - } - pStr->z[pStr->n] = 0; -} - -#if 0 /* NOT USED */ -/* -** Append nIndent space (0x20) characters to string *pStr. -*/ -static void lsmAppendIndent(LsmString *pStr, int nIndent){ - int i; - lsmStringExtend(pStr, nIndent); - for(i=0; i<nIndent; i++) lsmStringAppend(pStr, " ", 1); -} -#endif - -static void strAppendFlags(LsmString *pStr, u8 flags){ - char zFlags[8]; - - lsmFlagsToString(flags, zFlags); - zFlags[4] = ':'; - - lsmStringAppend(pStr, zFlags, 5); -} - -void dump_node_contents( - lsm_db *pDb, - u32 iNode, /* Print out the contents of this node */ - char *zPath, /* Path from root to this node */ - int nPath, /* Number of bytes in zPath */ - int nHeight /* Height: (0==leaf) (1==parent-of-leaf) */ -){ - const char *zSpace = " "; - int i; - int rc = LSM_OK; - LsmString s; - TreeNode *pNode; - TreeBlob b = {0, 0}; - - pNode = (TreeNode *)treeShmptr(pDb, iNode); - - if( nHeight==0 ){ - /* Append the nIndent bytes of space to string s. */ - lsmStringInit(&s, pDb->pEnv); - - /* Append each key to string s. */ - for(i=0; i<3; i++){ - u32 iPtr = pNode->aiKeyPtr[i]; - if( iPtr ){ - TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i],TKV_LOADKEY, &b,&rc); - strAppendFlags(&s, pKey->flags); - lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); - lsmStringAppend(&s, " ", -1); - } - } - - printf("% 6d %.*sleaf%.*s: %s\n", - iNode, nPath, zPath, 20-nPath-4, zSpace, s.z - ); - lsmStringClear(&s); - }else{ - for(i=0; i<4 && nHeight>0; i++){ - u32 iPtr = getChildPtr(pNode, pDb->treehdr.root.iTransId, i); - zPath[nPath] = (char)(i+'0'); - zPath[nPath+1] = '/'; - - if( iPtr ){ - dump_node_contents(pDb, iPtr, zPath, nPath+2, nHeight-1); - } - if( i!=3 && pNode->aiKeyPtr[i] ){ - TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i], TKV_LOADKEY,&b,&rc); - lsmStringInit(&s, pDb->pEnv); - strAppendFlags(&s, pKey->flags); - lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); - printf("% 6d %.*s%.*s: %s\n", - iNode, nPath+1, zPath, 20-nPath-1, zSpace, s.z); - lsmStringClear(&s); - } - } - } - - tblobFree(pDb, &b); -} - -void dump_tree_contents(lsm_db *pDb, const char *zCaption){ - char zPath[64]; - TreeRoot *p = &pDb->treehdr.root; - printf("\n%s\n", zCaption); - zPath[0] = '/'; - if( p->iRoot ){ - dump_node_contents(pDb, p->iRoot, zPath, 1, p->nHeight-1); - } - fflush(stdout); -} - -#endif - -/* -** Initialize a cursor object, the space for which has already been -** allocated. -*/ -static void treeCursorInit(lsm_db *pDb, int bOld, TreeCursor *pCsr){ - memset(pCsr, 0, sizeof(TreeCursor)); - pCsr->pDb = pDb; - if( bOld ){ - pCsr->pRoot = &pDb->treehdr.oldroot; - }else{ - pCsr->pRoot = &pDb->treehdr.root; - } - pCsr->iNode = -1; -} - -/* -** Return a pointer to the mapping of the TreeKey object that the cursor -** is pointing to. -*/ -static TreeKey *csrGetKey(TreeCursor *pCsr, TreeBlob *pBlob, int *pRc){ - TreeKey *pRet; - lsm_db *pDb = pCsr->pDb; - u32 iPtr = pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]]; - - assert( iPtr ); - pRet = (TreeKey*)treeShmptrUnsafe(pDb, iPtr); - if( !(pRet->flags & LSM_CONTIGUOUS) ){ - pRet = treeShmkey(pDb, iPtr, TKV_LOADVAL, pBlob, pRc); - } - - return pRet; -} - -/* -** Save the current position of tree cursor pCsr. -*/ -int lsmTreeCursorSave(TreeCursor *pCsr){ - int rc = LSM_OK; - if( pCsr && pCsr->pSave==0 ){ - int iNode = pCsr->iNode; - if( iNode>=0 ){ - pCsr->pSave = csrGetKey(pCsr, &pCsr->blob, &rc); - } - pCsr->iNode = -1; - } - return rc; -} - -/* -** Restore the position of a saved tree cursor. -*/ -static int treeCursorRestore(TreeCursor *pCsr, int *pRes){ - int rc = LSM_OK; - if( pCsr->pSave ){ - TreeKey *pKey = pCsr->pSave; - pCsr->pSave = 0; - if( pRes ){ - rc = lsmTreeCursorSeek(pCsr, TKV_KEY(pKey), pKey->nKey, pRes); - } - } - return rc; -} - -/* -** Allocate nByte bytes of space within the *-shm file. If successful, -** return LSM_OK and set *piPtr to the offset within the file at which -** the allocated space is located. -*/ -static u32 treeShmalloc(lsm_db *pDb, int bAlign, int nByte, int *pRc){ - u32 iRet = 0; - if( *pRc==LSM_OK ){ - const static int CHUNK_SIZE = LSM_SHM_CHUNK_SIZE; - const static int CHUNK_HDR = LSM_SHM_CHUNK_HDR; - u32 iWrite; /* Current write offset */ - u32 iEof; /* End of current chunk */ - int iChunk; /* Current chunk */ - - assert( nByte <= (CHUNK_SIZE-CHUNK_HDR) ); - - /* Check if there is enough space on the current chunk to fit the - ** new allocation. If not, link in a new chunk and put the new - ** allocation at the start of it. */ - iWrite = pDb->treehdr.iWrite; - if( bAlign ){ - iWrite = (iWrite + 3) & ~0x0003; - assert( (iWrite % 4)==0 ); - } - - assert( iWrite ); - iChunk = treeOffsetToChunk(iWrite-1); - iEof = (iChunk+1) * CHUNK_SIZE; - assert( iEof>=iWrite && (iEof-iWrite)<(u32)CHUNK_SIZE ); - if( (iWrite+nByte)>iEof ){ - ShmChunk *pHdr; /* Header of chunk just finished (iChunk) */ - ShmChunk *pFirst; /* Header of chunk treehdr.iFirst */ - ShmChunk *pNext; /* Header of new chunk */ - int iNext = 0; /* Next chunk */ - int rc = LSM_OK; - - pFirst = treeShmChunk(pDb, pDb->treehdr.iFirst); - - assert( shm_sequence_ge(pDb->treehdr.iUsedShmid, pFirst->iShmid) ); - assert( (pDb->treehdr.iNextShmid+1-pDb->treehdr.nChunk)==pFirst->iShmid ); - - /* Check if the chunk at the start of the linked list is still in - ** use. If not, reuse it. If so, allocate a new chunk by appending - ** to the *-shm file. */ - if( pDb->treehdr.iUsedShmid!=pFirst->iShmid ){ - int bInUse; - rc = lsmTreeInUse(pDb, pFirst->iShmid, &bInUse); - if( rc!=LSM_OK ){ - *pRc = rc; - return 0; - } - if( bInUse==0 ){ - iNext = pDb->treehdr.iFirst; - pDb->treehdr.iFirst = pFirst->iNext; - assert( pDb->treehdr.iFirst ); - } - } - if( iNext==0 ) iNext = pDb->treehdr.nChunk++; - - /* Set the header values for the new chunk */ - pNext = treeShmChunkRc(pDb, iNext, &rc); - if( pNext ){ - pNext->iNext = 0; - pNext->iShmid = (pDb->treehdr.iNextShmid++); - }else{ - *pRc = rc; - return 0; - } - - /* Set the header values for the chunk just finished */ - pHdr = (ShmChunk *)treeShmptr(pDb, iChunk*CHUNK_SIZE); - pHdr->iNext = iNext; - - /* Advance to the next chunk */ - iWrite = iNext * CHUNK_SIZE + CHUNK_HDR; - } - - /* Allocate space at iWrite. */ - iRet = iWrite; - pDb->treehdr.iWrite = iWrite + nByte; - pDb->treehdr.root.nByte += nByte; - } - return iRet; -} - -/* -** Allocate and zero nByte bytes of space within the *-shm file. -*/ -static void *treeShmallocZero(lsm_db *pDb, int nByte, u32 *piPtr, int *pRc){ - u32 iPtr; - void *p; - iPtr = treeShmalloc(pDb, 1, nByte, pRc); - p = treeShmptr(pDb, iPtr); - if( p ){ - assert( *pRc==LSM_OK ); - memset(p, 0, nByte); - *piPtr = iPtr; - } - return p; -} - -static TreeNode *newTreeNode(lsm_db *pDb, u32 *piPtr, int *pRc){ - return treeShmallocZero(pDb, sizeof(TreeNode), piPtr, pRc); -} - -static TreeLeaf *newTreeLeaf(lsm_db *pDb, u32 *piPtr, int *pRc){ - return treeShmallocZero(pDb, sizeof(TreeLeaf), piPtr, pRc); -} - -static TreeKey *newTreeKey( - lsm_db *pDb, - u32 *piPtr, - void *pKey, int nKey, /* Key data */ - void *pVal, int nVal, /* Value data (or nVal<0 for delete) */ - int *pRc -){ - TreeKey *p; - u32 iPtr; - u32 iEnd; - int nRem; - u8 *a; - int n; - - /* Allocate space for the TreeKey structure itself */ - *piPtr = iPtr = treeShmalloc(pDb, 1, sizeof(TreeKey), pRc); - p = treeShmptr(pDb, iPtr); - if( *pRc ) return 0; - p->nKey = nKey; - p->nValue = nVal; - - /* Allocate and populate the space required for the key and value. */ - n = nRem = nKey; - a = (u8 *)pKey; - while( a ){ - while( nRem>0 ){ - u8 *aAlloc; - int nAlloc; - u32 iWrite; - - iWrite = (pDb->treehdr.iWrite & (LSM_SHM_CHUNK_SIZE-1)); - iWrite = LSM_MAX(iWrite, LSM_SHM_CHUNK_HDR); - nAlloc = LSM_MIN((LSM_SHM_CHUNK_SIZE-iWrite), (u32)nRem); - - aAlloc = treeShmptr(pDb, treeShmalloc(pDb, 0, nAlloc, pRc)); - if( aAlloc==0 ) break; - memcpy(aAlloc, &a[n-nRem], nAlloc); - nRem -= nAlloc; - } - a = pVal; - n = nRem = nVal; - pVal = 0; - } - - iEnd = iPtr + sizeof(TreeKey) + nKey + LSM_MAX(0, nVal); - if( (iPtr & ~(LSM_SHM_CHUNK_SIZE-1))!=(iEnd & ~(LSM_SHM_CHUNK_SIZE-1)) ){ - p->flags = 0; - }else{ - p->flags = LSM_CONTIGUOUS; - } - - if( *pRc ) return 0; -#if 0 - printf("store: %d %s\n", (int)iPtr, (char *)pKey); -#endif - return p; -} - -static TreeNode *copyTreeNode( - lsm_db *pDb, - TreeNode *pOld, - u32 *piNew, - int *pRc -){ - TreeNode *pNew; - - pNew = newTreeNode(pDb, piNew, pRc); - if( pNew ){ - memcpy(pNew->aiKeyPtr, pOld->aiKeyPtr, sizeof(pNew->aiKeyPtr)); - memcpy(pNew->aiChildPtr, pOld->aiChildPtr, sizeof(pNew->aiChildPtr)); - if( pOld->iV2 ) pNew->aiChildPtr[pOld->iV2Child] = pOld->iV2Ptr; - } - return pNew; -} - -static TreeNode *copyTreeLeaf( - lsm_db *pDb, - TreeLeaf *pOld, - u32 *piNew, - int *pRc -){ - TreeLeaf *pNew; - pNew = newTreeLeaf(pDb, piNew, pRc); - if( pNew ){ - memcpy(pNew, pOld, sizeof(TreeLeaf)); - } - return (TreeNode *)pNew; -} - -/* -** The tree cursor passed as the second argument currently points to an -** internal node (not a leaf). Specifically, to a sub-tree pointer. This -** function replaces the sub-tree that the cursor currently points to -** with sub-tree pNew. -** -** The sub-tree may be replaced either by writing the "v2 data" on the -** internal node, or by allocating a new TreeNode structure and then -** calling this function on the parent of the internal node. -*/ -static int treeUpdatePtr(lsm_db *pDb, TreeCursor *pCsr, u32 iNew){ - int rc = LSM_OK; - if( pCsr->iNode<0 ){ - /* iNew is the new root node */ - pDb->treehdr.root.iRoot = iNew; - }else{ - /* If this node already has version 2 content, allocate a copy and - ** update the copy with the new pointer value. Otherwise, store the - ** new pointer as v2 data within the current node structure. */ - - TreeNode *p; /* The node to be modified */ - int iChildPtr; /* apChild[] entry to modify */ - - p = pCsr->apTreeNode[pCsr->iNode]; - iChildPtr = pCsr->aiCell[pCsr->iNode]; - - if( p->iV2 ){ - /* The "allocate new TreeNode" option */ - u32 iCopy; - TreeNode *pCopy; - pCopy = copyTreeNode(pDb, p, &iCopy, &rc); - if( pCopy ){ - assert( rc==LSM_OK ); - pCopy->aiChildPtr[iChildPtr] = iNew; - pCsr->iNode--; - rc = treeUpdatePtr(pDb, pCsr, iCopy); - } - }else{ - /* The "v2 data" option */ - u32 iPtr; - assert( pDb->treehdr.root.iTransId>0 ); - - if( pCsr->iNode ){ - iPtr = getChildPtr( - pCsr->apTreeNode[pCsr->iNode-1], - pDb->treehdr.root.iTransId, pCsr->aiCell[pCsr->iNode-1] - ); - }else{ - iPtr = pDb->treehdr.root.iRoot; - } - rc = intArrayAppend(pDb->pEnv, &pDb->rollback, iPtr); - - if( rc==LSM_OK ){ - p->iV2 = pDb->treehdr.root.iTransId; - p->iV2Child = (u8)iChildPtr; - p->iV2Ptr = iNew; - } - } - } - - return rc; -} - -/* -** Cursor pCsr points at a node that is part of pTree. This function -** inserts a new key and optionally child node pointer into that node. -** -** The position into which the new key and pointer are inserted is -** determined by the iSlot parameter. The new key will be inserted to -** the left of the key currently stored in apKey[iSlot]. Or, if iSlot is -** greater than the index of the rightmost key in the node. -** -** Pointer pLeftPtr points to a child tree that contains keys that are -** smaller than pTreeKey. -*/ -static int treeInsert( - lsm_db *pDb, /* Database handle */ - TreeCursor *pCsr, /* Cursor indicating path to insert at */ - u32 iLeftPtr, /* Left child pointer */ - u32 iTreeKey, /* Location of key to insert */ - u32 iRightPtr, /* Right child pointer */ - int iSlot /* Position to insert key into */ -){ - int rc = LSM_OK; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - - /* Check if the node is currently full. If so, split pNode in two and - ** call this function recursively to add a key to the parent. Otherwise, - ** insert the new key directly into pNode. */ - assert( pNode->aiKeyPtr[1] ); - if( pNode->aiKeyPtr[0] && pNode->aiKeyPtr[2] ){ - u32 iLeft; TreeNode *pLeft; /* New left-hand sibling node */ - u32 iRight; TreeNode *pRight; /* New right-hand sibling node */ - - pLeft = newTreeNode(pDb, &iLeft, &rc); - pRight = newTreeNode(pDb, &iRight, &rc); - if( rc ) return rc; - - pLeft->aiChildPtr[1] = getChildPtr(pNode, WORKING_VERSION, 0); - pLeft->aiKeyPtr[1] = pNode->aiKeyPtr[0]; - pLeft->aiChildPtr[2] = getChildPtr(pNode, WORKING_VERSION, 1); - - pRight->aiChildPtr[1] = getChildPtr(pNode, WORKING_VERSION, 2); - pRight->aiKeyPtr[1] = pNode->aiKeyPtr[2]; - pRight->aiChildPtr[2] = getChildPtr(pNode, WORKING_VERSION, 3); - - if( pCsr->iNode==0 ){ - /* pNode is the root of the tree. Grow the tree by one level. */ - u32 iRoot; TreeNode *pRoot; /* New root node */ - - pRoot = newTreeNode(pDb, &iRoot, &rc); - pRoot->aiKeyPtr[1] = pNode->aiKeyPtr[1]; - pRoot->aiChildPtr[1] = iLeft; - pRoot->aiChildPtr[2] = iRight; - - pDb->treehdr.root.iRoot = iRoot; - pDb->treehdr.root.nHeight++; - }else{ - - pCsr->iNode--; - rc = treeInsert(pDb, pCsr, - iLeft, pNode->aiKeyPtr[1], iRight, pCsr->aiCell[pCsr->iNode] - ); - } - - assert( pLeft->iV2==0 ); - assert( pRight->iV2==0 ); - switch( iSlot ){ - case 0: - pLeft->aiKeyPtr[0] = iTreeKey; - pLeft->aiChildPtr[0] = iLeftPtr; - if( iRightPtr ) pLeft->aiChildPtr[1] = iRightPtr; - break; - case 1: - pLeft->aiChildPtr[3] = (iRightPtr ? iRightPtr : pLeft->aiChildPtr[2]); - pLeft->aiKeyPtr[2] = iTreeKey; - pLeft->aiChildPtr[2] = iLeftPtr; - break; - case 2: - pRight->aiKeyPtr[0] = iTreeKey; - pRight->aiChildPtr[0] = iLeftPtr; - if( iRightPtr ) pRight->aiChildPtr[1] = iRightPtr; - break; - case 3: - pRight->aiChildPtr[3] = (iRightPtr ? iRightPtr : pRight->aiChildPtr[2]); - pRight->aiKeyPtr[2] = iTreeKey; - pRight->aiChildPtr[2] = iLeftPtr; - break; - } - - }else{ - TreeNode *pNew; - u32 *piKey; - u32 *piChild; - u32 iStore = 0; - u32 iNew = 0; - int i; - - /* Allocate a new version of node pNode. */ - pNew = newTreeNode(pDb, &iNew, &rc); - if( rc ) return rc; - - piKey = pNew->aiKeyPtr; - piChild = pNew->aiChildPtr; - - for(i=0; i<iSlot; i++){ - if( pNode->aiKeyPtr[i] ){ - *(piKey++) = pNode->aiKeyPtr[i]; - *(piChild++) = getChildPtr(pNode, WORKING_VERSION, i); - } - } - - *piKey++ = iTreeKey; - *piChild++ = iLeftPtr; - - iStore = iRightPtr; - for(i=iSlot; i<3; i++){ - if( pNode->aiKeyPtr[i] ){ - *(piKey++) = pNode->aiKeyPtr[i]; - *(piChild++) = iStore ? iStore : getChildPtr(pNode, WORKING_VERSION, i); - iStore = 0; - } - } - - if( iStore ){ - *piChild = iStore; - }else{ - *piChild = getChildPtr(pNode, WORKING_VERSION, - (pNode->aiKeyPtr[2] ? 3 : 2) - ); - } - pCsr->iNode--; - rc = treeUpdatePtr(pDb, pCsr, iNew); - } - - return rc; -} - -static int treeInsertLeaf( - lsm_db *pDb, /* Database handle */ - TreeCursor *pCsr, /* Cursor structure */ - u32 iTreeKey, /* Key pointer to insert */ - int iSlot /* Insert key to the left of this */ -){ - int rc = LSM_OK; /* Return code */ - TreeNode *pLeaf = pCsr->apTreeNode[pCsr->iNode]; - TreeLeaf *pNew; - u32 iNew; - - assert( iSlot>=0 && iSlot<=4 ); - assert( pCsr->iNode>0 ); - assert( pLeaf->aiKeyPtr[1] ); - - pCsr->iNode--; - - pNew = newTreeLeaf(pDb, &iNew, &rc); - if( pNew ){ - if( pLeaf->aiKeyPtr[0] && pLeaf->aiKeyPtr[2] ){ - /* The leaf is full. Split it in two. */ - TreeLeaf *pRight; - u32 iRight; - pRight = newTreeLeaf(pDb, &iRight, &rc); - if( pRight ){ - assert( rc==LSM_OK ); - pNew->aiKeyPtr[1] = pLeaf->aiKeyPtr[0]; - pRight->aiKeyPtr[1] = pLeaf->aiKeyPtr[2]; - switch( iSlot ){ - case 0: pNew->aiKeyPtr[0] = iTreeKey; break; - case 1: pNew->aiKeyPtr[2] = iTreeKey; break; - case 2: pRight->aiKeyPtr[0] = iTreeKey; break; - case 3: pRight->aiKeyPtr[2] = iTreeKey; break; - } - - rc = treeInsert(pDb, pCsr, iNew, pLeaf->aiKeyPtr[1], iRight, - pCsr->aiCell[pCsr->iNode] - ); - } - }else{ - int iOut = 0; - int i; - for(i=0; i<4; i++){ - if( i==iSlot ) pNew->aiKeyPtr[iOut++] = iTreeKey; - if( i<3 && pLeaf->aiKeyPtr[i] ){ - pNew->aiKeyPtr[iOut++] = pLeaf->aiKeyPtr[i]; - } - } - rc = treeUpdatePtr(pDb, pCsr, iNew); - } - } - - return rc; -} - -void lsmTreeMakeOld(lsm_db *pDb){ - - /* A write transaction must be open. Otherwise the code below that - ** assumes (pDb->pClient->iLogOff) is current may malfunction. - ** - ** Update: currently this assert fails due to lsm_flush(), which does - ** not set nTransOpen. - */ - assert( /* pDb->nTransOpen>0 && */ pDb->iReader>=0 ); - - if( pDb->treehdr.iOldShmid==0 ){ - pDb->treehdr.iOldLog = (pDb->treehdr.log.aRegion[2].iEnd << 1); - pDb->treehdr.iOldLog |= (~(pDb->pClient->iLogOff) & (i64)0x0001); - - pDb->treehdr.oldcksum0 = pDb->treehdr.log.cksum0; - pDb->treehdr.oldcksum1 = pDb->treehdr.log.cksum1; - pDb->treehdr.iOldShmid = pDb->treehdr.iNextShmid-1; - memcpy(&pDb->treehdr.oldroot, &pDb->treehdr.root, sizeof(TreeRoot)); - - pDb->treehdr.root.iTransId = 1; - pDb->treehdr.root.iRoot = 0; - pDb->treehdr.root.nHeight = 0; - pDb->treehdr.root.nByte = 0; - } -} - -void lsmTreeDiscardOld(lsm_db *pDb){ - assert( lsmShmAssertLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL) - || lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) - ); - pDb->treehdr.iUsedShmid = pDb->treehdr.iOldShmid; - pDb->treehdr.iOldShmid = 0; -} - -int lsmTreeHasOld(lsm_db *pDb){ - return pDb->treehdr.iOldShmid!=0; -} - -/* -** This function is called during recovery to initialize the -** tree header. Only the database connections private copy of the tree-header -** is initialized here - it will be copied into shared memory if log file -** recovery is successful. -*/ -int lsmTreeInit(lsm_db *pDb){ - ShmChunk *pOne; - int rc = LSM_OK; - - memset(&pDb->treehdr, 0, sizeof(TreeHeader)); - pDb->treehdr.root.iTransId = 1; - pDb->treehdr.iFirst = 1; - pDb->treehdr.nChunk = 2; - pDb->treehdr.iWrite = LSM_SHM_CHUNK_SIZE + LSM_SHM_CHUNK_HDR; - pDb->treehdr.iNextShmid = 2; - pDb->treehdr.iUsedShmid = 1; - - pOne = treeShmChunkRc(pDb, 1, &rc); - if( pOne ){ - pOne->iNext = 0; - pOne->iShmid = 1; - } - return rc; -} - -static void treeHeaderChecksum( - TreeHeader *pHdr, - u32 *aCksum -){ - u32 cksum1 = 0x12345678; - u32 cksum2 = 0x9ABCDEF0; - u32 *a = (u32 *)pHdr; - int i; - - assert( (offsetof(TreeHeader, aCksum) + sizeof(u32)*2)==sizeof(TreeHeader) ); - assert( (sizeof(TreeHeader) % (sizeof(u32)*2))==0 ); - - for(i=0; i<(offsetof(TreeHeader, aCksum) / sizeof(u32)); i+=2){ - cksum1 += a[i]; - cksum2 += (cksum1 + a[i+1]); - } - aCksum[0] = cksum1; - aCksum[1] = cksum2; -} - -/* -** Return true if the checksum stored in TreeHeader object *pHdr is -** consistent with the contents of its other fields. -*/ -static int treeHeaderChecksumOk(TreeHeader *pHdr){ - u32 aCksum[2]; - treeHeaderChecksum(pHdr, aCksum); - return (0==memcmp(aCksum, pHdr->aCksum, sizeof(aCksum))); -} - -/* -** This type is used by functions lsmTreeRepair() and treeSortByShmid() to -** make relinking the linked list of shared-memory chunks easier. -*/ -typedef struct ShmChunkLoc ShmChunkLoc; -struct ShmChunkLoc { - ShmChunk *pShm; - u32 iLoc; -}; - -/* -** This function checks that the linked list of shared memory chunks -** that starts at chunk db->treehdr.iFirst: -** -** 1) Includes all chunks in the shared-memory region, and -** 2) Links them together in order of ascending shm-id. -** -** If no error occurs and the conditions above are met, LSM_OK is returned. -** -** If either of the conditions are untrue, LSM_CORRUPT is returned. Or, if -** an error is encountered before the checks are completed, another LSM error -** code (i.e. LSM_IOERR or LSM_NOMEM) may be returned. -*/ -static int treeCheckLinkedList(lsm_db *db){ - int rc = LSM_OK; - int nVisit = 0; - ShmChunk *p; - - p = treeShmChunkRc(db, db->treehdr.iFirst, &rc); - while( rc==LSM_OK && p ){ - if( p->iNext ){ - if( p->iNext>=db->treehdr.nChunk ){ - rc = LSM_CORRUPT_BKPT; - }else{ - ShmChunk *pNext = treeShmChunkRc(db, p->iNext, &rc); - if( rc==LSM_OK ){ - if( pNext->iShmid!=p->iShmid+1 ){ - rc = LSM_CORRUPT_BKPT; - } - p = pNext; - } - } - }else{ - p = 0; - } - nVisit++; - } - - if( rc==LSM_OK && (u32)nVisit!=db->treehdr.nChunk-1 ){ - rc = LSM_CORRUPT_BKPT; - } - return rc; -} - -/* -** Iterate through the current in-memory tree. If there are any v2-pointers -** with transaction ids larger than db->treehdr.iTransId, zero them. -*/ -static int treeRepairPtrs(lsm_db *db){ - int rc = LSM_OK; - - if( db->treehdr.root.nHeight>1 ){ - TreeCursor csr; /* Cursor used to iterate through tree */ - u32 iTransId = db->treehdr.root.iTransId; - - /* Initialize the cursor structure. Also decrement the nHeight variable - ** in the tree-header. This will prevent the cursor from visiting any - ** leaf nodes. */ - db->treehdr.root.nHeight--; - treeCursorInit(db, 0, &csr); - - rc = lsmTreeCursorEnd(&csr, 0); - while( rc==LSM_OK && lsmTreeCursorValid(&csr) ){ - TreeNode *pNode = csr.apTreeNode[csr.iNode]; - if( pNode->iV2>iTransId ){ - pNode->iV2Child = 0; - pNode->iV2Ptr = 0; - pNode->iV2 = 0; - } - rc = lsmTreeCursorNext(&csr); - } - tblobFree(csr.pDb, &csr.blob); - - db->treehdr.root.nHeight++; - } - - return rc; -} - -static int treeRepairList(lsm_db *db){ - int rc = LSM_OK; - int i; - ShmChunk *p; - ShmChunk *pMin = 0; - u32 iMin = 0; - - /* Iterate through all shm chunks. Find the smallest shm-id present in - ** the shared-memory region. */ - for(i=1; rc==LSM_OK && (u32)i<db->treehdr.nChunk; i++){ - p = treeShmChunkRc(db, i, &rc); - if( p && (pMin==0 || shm_sequence_ge(pMin->iShmid, p->iShmid)) ){ - pMin = p; - iMin = i; - } - } - - /* Fix the shm-id values on any chunks with a shm-id greater than or - ** equal to treehdr.iNextShmid. Then do a merge-sort of all chunks to - ** fix the ShmChunk.iNext pointers. - */ - if( rc==LSM_OK ){ - int nSort; - int nByte; - u32 iPrevShmid; - ShmChunkLoc *aSort; - - /* Allocate space for a merge sort. */ - nSort = 1; - while( (u32)nSort < (db->treehdr.nChunk-1) ) nSort = nSort * 2; - nByte = sizeof(ShmChunkLoc) * nSort * 2; - aSort = lsmMallocZeroRc(db->pEnv, nByte, &rc); - iPrevShmid = pMin->iShmid; - - /* Fix all shm-ids, if required. */ - if( rc==LSM_OK ){ - iPrevShmid = pMin->iShmid-1; - for(i=1; (u32)i<db->treehdr.nChunk; i++){ - p = treeShmChunk(db, i); - aSort[i-1].pShm = p; - aSort[i-1].iLoc = i; - if( (u32)i!=db->treehdr.iFirst ){ - if( shm_sequence_ge(p->iShmid, db->treehdr.iNextShmid) ){ - p->iShmid = iPrevShmid--; - } - } - } - if( iMin!=db->treehdr.iFirst ){ - p = treeShmChunk(db, db->treehdr.iFirst); - p->iShmid = iPrevShmid; - } - } - - if( rc==LSM_OK ){ - ShmChunkLoc *aSpace = &aSort[nSort]; - for(i=0; i<nSort; i++){ - if( aSort[i].pShm ){ - assert( shm_sequence_ge(aSort[i].pShm->iShmid, iPrevShmid) ); - assert( aSpace[aSort[i].pShm->iShmid - iPrevShmid].pShm==0 ); - aSpace[aSort[i].pShm->iShmid - iPrevShmid] = aSort[i]; - } - } - - if( aSpace[nSort-1].pShm ) aSpace[nSort-1].pShm->iNext = 0; - for(i=0; i<nSort-1; i++){ - if( aSpace[i].pShm ){ - aSpace[i].pShm->iNext = aSpace[i+1].iLoc; - } - } - - rc = treeCheckLinkedList(db); - lsmFree(db->pEnv, aSort); - } - } - - return rc; -} - -/* -** This function is called as part of opening a write-transaction if the -** writer-flag is already set - indicating that the previous writer -** failed before ending its transaction. -*/ -int lsmTreeRepair(lsm_db *db){ - int rc = LSM_OK; - TreeHeader hdr; - ShmHeader *pHdr = db->pShmhdr; - - /* Ensure that the two tree-headers are consistent. Copy one over the other - ** if necessary. Prefer the data from a tree-header for which the checksum - ** computes. Or, if they both compute, prefer tree-header-1. */ - if( memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(TreeHeader)) ){ - if( treeHeaderChecksumOk(&pHdr->hdr1) ){ - memcpy(&pHdr->hdr2, &pHdr->hdr1, sizeof(TreeHeader)); - }else{ - memcpy(&pHdr->hdr1, &pHdr->hdr2, sizeof(TreeHeader)); - } - } - - /* Save the connections current copy of the tree-header. It will be - ** restored before returning. */ - memcpy(&hdr, &db->treehdr, sizeof(TreeHeader)); - - /* Walk the tree. Zero any v2 pointers with a transaction-id greater than - ** the transaction-id currently in the tree-headers. */ - rc = treeRepairPtrs(db); - - /* Repair the linked list of shared-memory chunks. */ - if( rc==LSM_OK ){ - rc = treeRepairList(db); - } - - memcpy(&db->treehdr, &hdr, sizeof(TreeHeader)); - return rc; -} - -static void treeOverwriteKey(lsm_db *db, TreeCursor *pCsr, u32 iKey, int *pRc){ - if( *pRc==LSM_OK ){ - TreeRoot *p = &db->treehdr.root; - TreeNode *pNew; - u32 iNew; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - int iCell = pCsr->aiCell[pCsr->iNode]; - - /* Create a copy of this node */ - if( (pCsr->iNode>0 && (u32)pCsr->iNode==(p->nHeight-1)) ){ - pNew = copyTreeLeaf(db, (TreeLeaf *)pNode, &iNew, pRc); - }else{ - pNew = copyTreeNode(db, pNode, &iNew, pRc); - } - - if( pNew ){ - /* Modify the value in the new version */ - pNew->aiKeyPtr[iCell] = iKey; - - /* Change the pointer in the parent (if any) to point at the new - ** TreeNode */ - pCsr->iNode--; - treeUpdatePtr(db, pCsr, iNew); - } - } -} - -static int treeNextIsEndDelete(lsm_db *db, TreeCursor *pCsr){ - int iNode = pCsr->iNode; - int iCell = pCsr->aiCell[iNode]+1; - - /* Cursor currently points to a leaf node. */ - assert( (u32)pCsr->iNode==(db->treehdr.root.nHeight-1) ); - - while( iNode>=0 ){ - TreeNode *pNode = pCsr->apTreeNode[iNode]; - if( iCell<3 && pNode->aiKeyPtr[iCell] ){ - int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); - assert( rc==LSM_OK ); - return ((pKey->flags & LSM_END_DELETE) ? 1 : 0); - } - iNode--; - iCell = pCsr->aiCell[iNode]; - } - - return 0; -} - -static int treePrevIsStartDelete(lsm_db *db, TreeCursor *pCsr){ - int iNode = pCsr->iNode; - - /* Cursor currently points to a leaf node. */ - assert( (u32)pCsr->iNode==(db->treehdr.root.nHeight-1) ); - - while( iNode>=0 ){ - TreeNode *pNode = pCsr->apTreeNode[iNode]; - int iCell = pCsr->aiCell[iNode]-1; - if( iCell>=0 && pNode->aiKeyPtr[iCell] ){ - int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); - assert( rc==LSM_OK ); - return ((pKey->flags & LSM_START_DELETE) ? 1 : 0); - } - iNode--; - } - - return 0; -} - - -static int treeInsertEntry( - lsm_db *pDb, /* Database handle */ - int flags, /* Flags associated with entry */ - void *pKey, /* Pointer to key data */ - int nKey, /* Size of key data in bytes */ - void *pVal, /* Pointer to value data (or NULL) */ - int nVal /* Bytes in value data (or -ve for delete) */ -){ - int rc = LSM_OK; /* Return Code */ - TreeKey *pTreeKey; /* New key-value being inserted */ - u32 iTreeKey; - TreeRoot *p = &pDb->treehdr.root; - TreeCursor csr; /* Cursor to seek to pKey/nKey */ - int res = 0; /* Result of seek operation on csr */ - - assert( nVal>=0 || pVal==0 ); - assert_tree_looks_ok(LSM_OK, pTree); - assert( flags==LSM_INSERT || flags==LSM_POINT_DELETE - || flags==LSM_START_DELETE || flags==LSM_END_DELETE - ); - assert( (flags & LSM_CONTIGUOUS)==0 ); -#if 0 - dump_tree_contents(pDb, "before"); -#endif - - if( p->iRoot ){ - TreeKey *pRes; /* Key at end of seek operation */ - treeCursorInit(pDb, 0, &csr); - - /* Seek to the leaf (or internal node) that the new key belongs on */ - rc = lsmTreeCursorSeek(&csr, pKey, nKey, &res); - pRes = csrGetKey(&csr, &csr.blob, &rc); - if( rc!=LSM_OK ) return rc; - assert( pRes ); - - if( flags==LSM_START_DELETE ){ - /* When inserting a start-delete-range entry, if the key that - ** occurs immediately before the new entry is already a START_DELETE, - ** then the new entry is not required. */ - if( (res<=0 && (pRes->flags & LSM_START_DELETE)) - || (res>0 && treePrevIsStartDelete(pDb, &csr)) - ){ - goto insert_entry_out; - } - }else if( flags==LSM_END_DELETE ){ - /* When inserting an start-delete-range entry, if the key that - ** occurs immediately after the new entry is already an END_DELETE, - ** then the new entry is not required. */ - if( (res<0 && treeNextIsEndDelete(pDb, &csr)) - || (res>=0 && (pRes->flags & LSM_END_DELETE)) - ){ - goto insert_entry_out; - } - } - - if( res==0 && (flags & (LSM_END_DELETE|LSM_START_DELETE)) ){ - if( pRes->flags & LSM_INSERT ){ - nVal = pRes->nValue; - pVal = TKV_VAL(pRes); - } - flags = flags | pRes->flags; - } - - if( flags & (LSM_INSERT|LSM_POINT_DELETE) ){ - if( (res<0 && (pRes->flags & LSM_START_DELETE)) - || (res>0 && (pRes->flags & LSM_END_DELETE)) - ){ - flags = flags | (LSM_END_DELETE|LSM_START_DELETE); - }else if( res==0 ){ - flags = flags | (pRes->flags & (LSM_END_DELETE|LSM_START_DELETE)); - } - } - }else{ - memset(&csr, 0, sizeof(TreeCursor)); - } - - /* Allocate and populate a new key-value pair structure */ - pTreeKey = newTreeKey(pDb, &iTreeKey, pKey, nKey, pVal, nVal, &rc); - if( rc!=LSM_OK ) return rc; - assert( pTreeKey->flags==0 || pTreeKey->flags==LSM_CONTIGUOUS ); - pTreeKey->flags |= flags; - - if( p->iRoot==0 ){ - /* The tree is completely empty. Add a new root node and install - ** (pKey/nKey) as the middle entry. Even though it is a leaf at the - ** moment, use newTreeNode() to allocate the node (i.e. allocate enough - ** space for the fields used by interior nodes). This is because the - ** treeInsert() routine may convert this node to an interior node. */ - TreeNode *pRoot = newTreeNode(pDb, &p->iRoot, &rc); - if( rc==LSM_OK ){ - assert( p->nHeight==0 ); - pRoot->aiKeyPtr[1] = iTreeKey; - p->nHeight = 1; - } - }else{ - if( res==0 ){ - /* The search found a match within the tree. */ - treeOverwriteKey(pDb, &csr, iTreeKey, &rc); - }else{ - /* The cursor now points to the leaf node into which the new entry should - ** be inserted. There may or may not be a free slot within the leaf for - ** the new key-value pair. - ** - ** iSlot is set to the index of the key within pLeaf that the new key - ** should be inserted to the left of (or to a value 1 greater than the - ** index of the rightmost key if the new key is larger than all keys - ** currently stored in the node). - */ - int iSlot = csr.aiCell[csr.iNode] + (res<0); - if( csr.iNode==0 ){ - rc = treeInsert(pDb, &csr, 0, iTreeKey, 0, iSlot); - }else{ - rc = treeInsertLeaf(pDb, &csr, iTreeKey, iSlot); - } - } - } - -#if 0 - dump_tree_contents(pDb, "after"); -#endif - insert_entry_out: - tblobFree(pDb, &csr.blob); - assert_tree_looks_ok(rc, pTree); - return rc; -} - -/* -** Insert a new entry into the in-memory tree. -** -** If the value of the 5th parameter, nVal, is negative, then a delete-marker -** is inserted into the tree. In this case the value pointer, pVal, must be -** NULL. -*/ -int lsmTreeInsert( - lsm_db *pDb, /* Database handle */ - void *pKey, /* Pointer to key data */ - int nKey, /* Size of key data in bytes */ - void *pVal, /* Pointer to value data (or NULL) */ - int nVal /* Bytes in value data (or -ve for delete) */ -){ - int flags; - if( nVal<0 ){ - flags = LSM_POINT_DELETE; - }else{ - flags = LSM_INSERT; - } - - return treeInsertEntry(pDb, flags, pKey, nKey, pVal, nVal); -} - -static int treeDeleteEntry(lsm_db *db, TreeCursor *pCsr, u32 iNewptr){ - TreeRoot *p = &db->treehdr.root; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - int iSlot = pCsr->aiCell[pCsr->iNode]; - int bLeaf; - int rc = LSM_OK; - - assert( pNode->aiKeyPtr[1] ); - assert( pNode->aiKeyPtr[iSlot] ); - assert( iSlot==0 || iSlot==1 || iSlot==2 ); - assert( ((u32)pCsr->iNode==(db->treehdr.root.nHeight-1))==(iNewptr==0) ); - - bLeaf = ((u32)pCsr->iNode==(p->nHeight-1) && p->nHeight>1); - - if( pNode->aiKeyPtr[0] || pNode->aiKeyPtr[2] ){ - /* There are currently at least 2 keys on this node. So just create - ** a new copy of the node with one of the keys removed. If the node - ** happens to be the root node of the tree, allocate an entire - ** TreeNode structure instead of just a TreeLeaf. */ - TreeNode *pNew; - u32 iNew; - - if( bLeaf ){ - pNew = (TreeNode *)newTreeLeaf(db, &iNew, &rc); - }else{ - pNew = newTreeNode(db, &iNew, &rc); - } - if( pNew ){ - int i; - int iOut = 1; - for(i=0; i<4; i++){ - if( i==iSlot ){ - i++; - if( bLeaf==0 ) pNew->aiChildPtr[iOut] = iNewptr; - if( i<3 ) pNew->aiKeyPtr[iOut] = pNode->aiKeyPtr[i]; - iOut++; - }else if( bLeaf || p->nHeight==1 ){ - if( i<3 && pNode->aiKeyPtr[i] ){ - pNew->aiKeyPtr[iOut++] = pNode->aiKeyPtr[i]; - } - }else{ - if( getChildPtr(pNode, WORKING_VERSION, i) ){ - pNew->aiChildPtr[iOut] = getChildPtr(pNode, WORKING_VERSION, i); - if( i<3 ) pNew->aiKeyPtr[iOut] = pNode->aiKeyPtr[i]; - iOut++; - } - } - } - assert( iOut<=4 ); - assert( bLeaf || pNew->aiChildPtr[0]==0 ); - pCsr->iNode--; - rc = treeUpdatePtr(db, pCsr, iNew); - } - - }else if( pCsr->iNode==0 ){ - /* Removing the only key in the root node. iNewptr is the new root. */ - assert( iSlot==1 ); - db->treehdr.root.iRoot = iNewptr; - db->treehdr.root.nHeight--; - - }else{ - /* There is only one key on this node and the node is not the root - ** node. Find a peer for this node. Then redistribute the contents of - ** the peer and the parent cell between the parent and either one or - ** two new nodes. */ - TreeNode *pParent; /* Parent tree node */ - int iPSlot; - u32 iPeer; /* Pointer to peer leaf node */ - int iDir; - TreeNode *pPeer; /* The peer leaf node */ - TreeNode *pNew1; u32 iNew1; /* First new leaf node */ - - assert( iSlot==1 ); - - pParent = pCsr->apTreeNode[pCsr->iNode-1]; - iPSlot = pCsr->aiCell[pCsr->iNode-1]; - - if( iPSlot>0 && getChildPtr(pParent, WORKING_VERSION, iPSlot-1) ){ - iDir = -1; - }else{ - iDir = +1; - } - iPeer = getChildPtr(pParent, WORKING_VERSION, iPSlot+iDir); - pPeer = (TreeNode *)treeShmptr(db, iPeer); - assertIsWorkingChild(db, pNode, pParent, iPSlot); - - /* Allocate the first new leaf node. This is always required. */ - if( bLeaf ){ - pNew1 = (TreeNode *)newTreeLeaf(db, &iNew1, &rc); - }else{ - pNew1 = (TreeNode *)newTreeNode(db, &iNew1, &rc); - } - - if( pPeer->aiKeyPtr[0] && pPeer->aiKeyPtr[2] ){ - /* Peer node is completely full. This means that two new leaf nodes - ** and a new parent node are required. */ - - TreeNode *pNew2; u32 iNew2; /* Second new leaf node */ - TreeNode *pNewP; u32 iNewP; /* New parent node */ - - if( bLeaf ){ - pNew2 = (TreeNode *)newTreeLeaf(db, &iNew2, &rc); - }else{ - pNew2 = (TreeNode *)newTreeNode(db, &iNew2, &rc); - } - pNewP = copyTreeNode(db, pParent, &iNewP, &rc); - - if( iDir==-1 ){ - pNew1->aiKeyPtr[1] = pPeer->aiKeyPtr[0]; - if( bLeaf==0 ){ - pNew1->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 0); - pNew1->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 1); - } - - pNewP->aiChildPtr[iPSlot-1] = iNew1; - pNewP->aiKeyPtr[iPSlot-1] = pPeer->aiKeyPtr[1]; - pNewP->aiChildPtr[iPSlot] = iNew2; - - pNew2->aiKeyPtr[0] = pPeer->aiKeyPtr[2]; - pNew2->aiKeyPtr[1] = pParent->aiKeyPtr[iPSlot-1]; - if( bLeaf==0 ){ - pNew2->aiChildPtr[0] = getChildPtr(pPeer, WORKING_VERSION, 2); - pNew2->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 3); - pNew2->aiChildPtr[2] = iNewptr; - } - }else{ - pNew1->aiKeyPtr[1] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ){ - pNew1->aiChildPtr[1] = iNewptr; - pNew1->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 0); - } - - pNewP->aiChildPtr[iPSlot] = iNew1; - pNewP->aiKeyPtr[iPSlot] = pPeer->aiKeyPtr[0]; - pNewP->aiChildPtr[iPSlot+1] = iNew2; - - pNew2->aiKeyPtr[0] = pPeer->aiKeyPtr[1]; - pNew2->aiKeyPtr[1] = pPeer->aiKeyPtr[2]; - if( bLeaf==0 ){ - pNew2->aiChildPtr[0] = getChildPtr(pPeer, WORKING_VERSION, 1); - pNew2->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 2); - pNew2->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 3); - } - } - assert( pCsr->iNode>=1 ); - pCsr->iNode -= 2; - if( rc==LSM_OK ){ - assert( pNew1->aiKeyPtr[1] && pNew2->aiKeyPtr[1] ); - rc = treeUpdatePtr(db, pCsr, iNewP); - } - }else{ - int iKOut = 0; - int iPOut = 0; - int i; - - pCsr->iNode--; - - if( iDir==1 ){ - pNew1->aiKeyPtr[iKOut++] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ) pNew1->aiChildPtr[iPOut++] = iNewptr; - } - for(i=0; i<3; i++){ - if( pPeer->aiKeyPtr[i] ){ - pNew1->aiKeyPtr[iKOut++] = pPeer->aiKeyPtr[i]; - } - } - if( bLeaf==0 ){ - for(i=0; i<4; i++){ - if( getChildPtr(pPeer, WORKING_VERSION, i) ){ - pNew1->aiChildPtr[iPOut++] = getChildPtr(pPeer, WORKING_VERSION, i); - } - } - } - if( iDir==-1 ){ - iPSlot--; - pNew1->aiKeyPtr[iKOut++] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ) pNew1->aiChildPtr[iPOut++] = iNewptr; - pCsr->aiCell[pCsr->iNode] = (u8)iPSlot; - } - - rc = treeDeleteEntry(db, pCsr, iNew1); - } - } - - return rc; -} - -/* -** Delete a range of keys from the tree structure (i.e. the lsm_delete_range() -** function, not lsm_delete()). -** -** This is a two step process: -** -** 1) Remove all entries currently stored in the tree that have keys -** that fall into the deleted range. -** -** TODO: There are surely good ways to optimize this step - removing -** a range of keys from a b-tree. But for now, this function removes -** them one at a time using the usual approach. -** -** 2) Unless the largest key smaller than or equal to (pKey1/nKey1) is -** already marked as START_DELETE, insert a START_DELETE key. -** Similarly, unless the smallest key greater than or equal to -** (pKey2/nKey2) is already START_END, insert a START_END key. -*/ -int lsmTreeDelete( - lsm_db *db, - void *pKey1, int nKey1, /* Start of range */ - void *pKey2, int nKey2 /* End of range */ -){ - int rc = LSM_OK; - int bDone = 0; - TreeRoot *p = &db->treehdr.root; - TreeBlob blob = {0, 0}; - - /* The range must be sensible - that (key1 < key2). */ - assert( treeKeycmp(pKey1, nKey1, pKey2, nKey2)<0 ); - assert( assert_delete_ranges_match(db) ); - -#if 0 - static int nCall = 0; - printf("\n"); - nCall++; - printf("%d delete %s .. %s\n", nCall, (char *)pKey1, (char *)pKey2); - dump_tree_contents(db, "before delete"); -#endif - - /* Step 1. This loop runs until the tree contains no keys within the - ** range being deleted. Or until an error occurs. */ - while( bDone==0 && rc==LSM_OK ){ - int res; - TreeCursor csr; /* Cursor to seek to first key in range */ - void *pDel; int nDel; /* Key to (possibly) delete this iteration */ -#ifndef NDEBUG - int nEntry = treeCountEntries(db); -#endif - - /* Seek the cursor to the first entry in the tree greater than pKey1. */ - treeCursorInit(db, 0, &csr); - lsmTreeCursorSeek(&csr, pKey1, nKey1, &res); - if( res<=0 && lsmTreeCursorValid(&csr) ) lsmTreeCursorNext(&csr); - - /* If there is no such entry, or if it is greater than pKey2, then the - ** tree now contains no keys in the range being deleted. In this case - ** break out of the loop. */ - bDone = 1; - if( lsmTreeCursorValid(&csr) ){ - lsmTreeCursorKey(&csr, 0, &pDel, &nDel); - if( treeKeycmp(pDel, nDel, pKey2, nKey2)<0 ) bDone = 0; - } - - if( bDone==0 ){ - if( (u32)csr.iNode==(p->nHeight-1) ){ - /* The element to delete already lies on a leaf node */ - rc = treeDeleteEntry(db, &csr, 0); - }else{ - /* 1. Overwrite the current key with a copy of the next key in the - ** tree (key N). - ** - ** 2. Seek to key N (cursor will stop at the internal node copy of - ** N). Move to the next key (original copy of N). Delete - ** this entry. - */ - u32 iKey; - TreeKey *pKey; - int iNode = csr.iNode; - lsmTreeCursorNext(&csr); - assert( (u32)csr.iNode==(p->nHeight-1) ); - - iKey = csr.apTreeNode[csr.iNode]->aiKeyPtr[csr.aiCell[csr.iNode]]; - lsmTreeCursorPrev(&csr); - - treeOverwriteKey(db, &csr, iKey, &rc); - pKey = treeShmkey(db, iKey, TKV_LOADKEY, &blob, &rc); - if( pKey ){ - rc = lsmTreeCursorSeek(&csr, TKV_KEY(pKey), pKey->nKey, &res); - } - if( rc==LSM_OK ){ - assert( res==0 && csr.iNode==iNode ); - rc = lsmTreeCursorNext(&csr); - if( rc==LSM_OK ){ - rc = treeDeleteEntry(db, &csr, 0); - } - } - } - } - - /* Clean up any memory allocated by the cursor. */ - tblobFree(db, &csr.blob); -#if 0 - dump_tree_contents(db, "ddd delete"); -#endif - assert( bDone || treeCountEntries(db)==(nEntry-1) ); - } - -#if 0 - dump_tree_contents(db, "during delete"); -#endif - - /* Now insert the START_DELETE and END_DELETE keys. */ - if( rc==LSM_OK ){ - rc = treeInsertEntry(db, LSM_START_DELETE, pKey1, nKey1, 0, -1); - } -#if 0 - dump_tree_contents(db, "during delete 2"); -#endif - if( rc==LSM_OK ){ - rc = treeInsertEntry(db, LSM_END_DELETE, pKey2, nKey2, 0, -1); - } - -#if 0 - dump_tree_contents(db, "after delete"); -#endif - - tblobFree(db, &blob); - assert( assert_delete_ranges_match(db) ); - return rc; -} - -/* -** Return, in bytes, the amount of memory currently used by the tree -** structure. -*/ -int lsmTreeSize(lsm_db *pDb){ - return pDb->treehdr.root.nByte; -} - -/* -** Open a cursor on the in-memory tree pTree. -*/ -int lsmTreeCursorNew(lsm_db *pDb, int bOld, TreeCursor **ppCsr){ - TreeCursor *pCsr; - *ppCsr = pCsr = lsmMalloc(pDb->pEnv, sizeof(TreeCursor)); - if( pCsr ){ - treeCursorInit(pDb, bOld, pCsr); - return LSM_OK; - } - return LSM_NOMEM_BKPT; -} - -/* -** Close an in-memory tree cursor. -*/ -void lsmTreeCursorDestroy(TreeCursor *pCsr){ - if( pCsr ){ - tblobFree(pCsr->pDb, &pCsr->blob); - lsmFree(pCsr->pDb->pEnv, pCsr); - } -} - -void lsmTreeCursorReset(TreeCursor *pCsr){ - if( pCsr ){ - pCsr->iNode = -1; - pCsr->pSave = 0; - } -} - -#ifndef NDEBUG -static int treeCsrCompare(TreeCursor *pCsr, void *pKey, int nKey, int *pRc){ - TreeKey *p; - int cmp = 0; - assert( pCsr->iNode>=0 ); - p = csrGetKey(pCsr, &pCsr->blob, pRc); - if( p ){ - cmp = treeKeycmp(TKV_KEY(p), p->nKey, pKey, nKey); - } - return cmp; -} -#endif - - -/* -** Attempt to seek the cursor passed as the first argument to key (pKey/nKey) -** in the tree structure. If an exact match for the key is found, leave the -** cursor pointing to it and set *pRes to zero before returning. If an -** exact match cannot be found, do one of the following: -** -** * Leave the cursor pointing to the smallest element in the tree that -** is larger than the key and set *pRes to +1, or -** -** * Leave the cursor pointing to the largest element in the tree that -** is smaller than the key and set *pRes to -1, or -** -** * If the tree is empty, leave the cursor at EOF and set *pRes to -1. -*/ -int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes){ - int rc = LSM_OK; /* Return code */ - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - u32 iNodePtr; /* Location of current node in search */ - - /* Discard any saved position data */ - treeCursorRestore(pCsr, 0); - - iNodePtr = pRoot->iRoot; - if( iNodePtr==0 ){ - /* Either an error occurred or the tree is completely empty. */ - assert( rc!=LSM_OK || pRoot->iRoot==0 ); - *pRes = -1; - pCsr->iNode = -1; - }else{ - TreeBlob b = {0, 0}; - int res = 0; /* Result of comparison function */ - int iNode = -1; - while( iNodePtr ){ - TreeNode *pNode; /* Node at location iNodePtr */ - int iTest; /* Index of second key to test (0 or 2) */ - u32 iTreeKey; - TreeKey *pTreeKey; /* Key to compare against */ - - pNode = (TreeNode *)treeShmptrUnsafe(pDb, iNodePtr); - iNode++; - pCsr->apTreeNode[iNode] = pNode; - - /* Compare (pKey/nKey) with the key in the middle slot of B-tree node - ** pNode. The middle slot is never empty. If the comparison is a match, - ** then the search is finished. Break out of the loop. */ - pTreeKey = (TreeKey*)treeShmptrUnsafe(pDb, pNode->aiKeyPtr[1]); - if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ - pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[1], TKV_LOADKEY, &b, &rc); - if( rc!=LSM_OK ) break; - } - res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); - if( res==0 ){ - pCsr->aiCell[iNode] = 1; - break; - } - - /* Based on the results of the previous comparison, compare (pKey/nKey) - ** to either the left or right key of the B-tree node, if such a key - ** exists. */ - iTest = (res>0 ? 0 : 2); - iTreeKey = pNode->aiKeyPtr[iTest]; - if( iTreeKey ){ - pTreeKey = (TreeKey*)treeShmptrUnsafe(pDb, iTreeKey); - if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ - pTreeKey = treeShmkey(pDb, iTreeKey, TKV_LOADKEY, &b, &rc); - if( rc ) break; - } - res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); - if( res==0 ){ - pCsr->aiCell[iNode] = (u8)iTest; - break; - } - }else{ - iTest = 1; - } - - if( (u32)iNode<(pRoot->nHeight-1) ){ - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iTest + (res<0)); - }else{ - iNodePtr = 0; - } - pCsr->aiCell[iNode] = (u8)(iTest + (iNodePtr && (res<0))); - } - - *pRes = res; - pCsr->iNode = iNode; - tblobFree(pDb, &b); - } - - /* assert() that *pRes has been set properly */ -#ifndef NDEBUG - if( rc==LSM_OK && lsmTreeCursorValid(pCsr) ){ - int cmp = treeCsrCompare(pCsr, pKey, nKey, &rc); - assert( rc!=LSM_OK || *pRes==cmp || (*pRes ^ cmp)>0 ); - } -#endif - - return rc; -} - -int lsmTreeCursorNext(TreeCursor *pCsr){ -#ifndef NDEBUG - TreeKey *pK1; - TreeBlob key1 = {0, 0}; -#endif - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - const int iLeaf = pRoot->nHeight-1; - int iCell; - int rc = LSM_OK; - TreeNode *pNode; - - /* Restore the cursor position, if required */ - int iRestore = 0; - treeCursorRestore(pCsr, &iRestore); - if( iRestore>0 ) return LSM_OK; - - /* Save a pointer to the current key. This is used in an assert() at the - ** end of this function - to check that the 'next' key really is larger - ** than the current key. */ -#ifndef NDEBUG - pK1 = csrGetKey(pCsr, &key1, &rc); - if( rc!=LSM_OK ) return rc; -#endif - - assert( lsmTreeCursorValid(pCsr) ); - assert( pCsr->aiCell[pCsr->iNode]<3 ); - - pNode = pCsr->apTreeNode[pCsr->iNode]; - iCell = ++pCsr->aiCell[pCsr->iNode]; - - /* If the current node is not a leaf, and the current cell has sub-tree - ** associated with it, descend to the left-most key on the left-most - ** leaf of the sub-tree. */ - if( pCsr->iNode<iLeaf && getChildPtr(pNode, pRoot->iTransId, iCell) ){ - do { - u32 iNodePtr; - pCsr->iNode++; - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - pCsr->apTreeNode[pCsr->iNode] = pNode; - iCell = pCsr->aiCell[pCsr->iNode] = (pNode->aiKeyPtr[0]==0); - }while( pCsr->iNode < iLeaf ); - } - - /* Otherwise, the next key is found by following pointer up the tree - ** until there is a key immediately to the right of the pointer followed - ** to reach the sub-tree containing the current key. */ - else if( iCell>=3 || pNode->aiKeyPtr[iCell]==0 ){ - while( (--pCsr->iNode)>=0 ){ - iCell = pCsr->aiCell[pCsr->iNode]; - if( iCell<3 && pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[iCell] ) break; - } - } - -#ifndef NDEBUG - if( pCsr->iNode>=0 ){ - TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc||treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)>=0 ); - } - tblobFree(pDb, &key1); -#endif - - return rc; -} - -int lsmTreeCursorPrev(TreeCursor *pCsr){ -#ifndef NDEBUG - TreeKey *pK1; - TreeBlob key1 = {0, 0}; -#endif - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - const int iLeaf = pRoot->nHeight-1; - int iCell; - int rc = LSM_OK; - TreeNode *pNode; - - /* Restore the cursor position, if required */ - int iRestore = 0; - treeCursorRestore(pCsr, &iRestore); - if( iRestore<0 ) return LSM_OK; - - /* Save a pointer to the current key. This is used in an assert() at the - ** end of this function - to check that the 'next' key really is smaller - ** than the current key. */ -#ifndef NDEBUG - pK1 = csrGetKey(pCsr, &key1, &rc); - if( rc!=LSM_OK ) return rc; -#endif - - assert( lsmTreeCursorValid(pCsr) ); - pNode = pCsr->apTreeNode[pCsr->iNode]; - iCell = pCsr->aiCell[pCsr->iNode]; - assert( iCell>=0 && iCell<3 ); - - /* If the current node is not a leaf, and the current cell has sub-tree - ** associated with it, descend to the right-most key on the right-most - ** leaf of the sub-tree. */ - if( pCsr->iNode<iLeaf && getChildPtr(pNode, pRoot->iTransId, iCell) ){ - do { - u32 iNodePtr; - pCsr->iNode++; - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - if( rc!=LSM_OK ) break; - pCsr->apTreeNode[pCsr->iNode] = pNode; - iCell = 1 + (pNode->aiKeyPtr[2]!=0) + (pCsr->iNode < iLeaf); - pCsr->aiCell[pCsr->iNode] = (u8)iCell; - }while( pCsr->iNode < iLeaf ); - } - - /* Otherwise, the next key is found by following pointer up the tree until - ** there is a key immediately to the left of the pointer followed to reach - ** the sub-tree containing the current key. */ - else{ - do { - iCell = pCsr->aiCell[pCsr->iNode]-1; - if( iCell>=0 && pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[iCell] ) break; - }while( (--pCsr->iNode)>=0 ); - pCsr->aiCell[pCsr->iNode] = (u8)iCell; - } - -#ifndef NDEBUG - if( pCsr->iNode>=0 ){ - TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc || treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)<0 ); - } - tblobFree(pDb, &key1); -#endif - - return rc; -} - -/* -** Move the cursor to the first (bLast==0) or last (bLast!=0) entry in the -** in-memory tree. -*/ -int lsmTreeCursorEnd(TreeCursor *pCsr, int bLast){ - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - int rc = LSM_OK; - - u32 iNodePtr; - pCsr->iNode = -1; - - /* Discard any saved position data */ - treeCursorRestore(pCsr, 0); - - iNodePtr = pRoot->iRoot; - while( iNodePtr ){ - int iCell; - TreeNode *pNode; - - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - if( rc ) break; - - if( bLast ){ - iCell = ((pNode->aiKeyPtr[2]==0) ? 2 : 3); - }else{ - iCell = ((pNode->aiKeyPtr[0]==0) ? 1 : 0); - } - pCsr->iNode++; - pCsr->apTreeNode[pCsr->iNode] = pNode; - - if( (u32)pCsr->iNode<pRoot->nHeight-1 ){ - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - }else{ - iNodePtr = 0; - } - pCsr->aiCell[pCsr->iNode] = (u8)(iCell - (iNodePtr==0 && bLast)); - } - - return rc; -} - -int lsmTreeCursorFlags(TreeCursor *pCsr){ - int flags = 0; - if( pCsr && pCsr->iNode>=0 ){ - int rc = LSM_OK; - TreeKey *pKey = (TreeKey *)treeShmptrUnsafe(pCsr->pDb, - pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]] - ); - assert( rc==LSM_OK ); - flags = (pKey->flags & ~LSM_CONTIGUOUS); - } - return flags; -} - -int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey){ - TreeKey *pTreeKey; - int rc = LSM_OK; - - assert( lsmTreeCursorValid(pCsr) ); - - pTreeKey = pCsr->pSave; - if( !pTreeKey ){ - pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); - } - if( rc==LSM_OK ){ - *pnKey = pTreeKey->nKey; - if( pFlags ) *pFlags = pTreeKey->flags; - *ppKey = (void *)&pTreeKey[1]; - } - - return rc; -} - -int lsmTreeCursorValue(TreeCursor *pCsr, void **ppVal, int *pnVal){ - int res = 0; - int rc; - - rc = treeCursorRestore(pCsr, &res); - if( res==0 ){ - TreeKey *pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); - if( rc==LSM_OK ){ - if( pTreeKey->flags & LSM_INSERT ){ - *pnVal = pTreeKey->nValue; - *ppVal = TKV_VAL(pTreeKey); - }else{ - *ppVal = 0; - *pnVal = -1; - } - } - }else{ - *ppVal = 0; - *pnVal = 0; - } - - return rc; -} - -/* -** Return true if the cursor currently points to a valid entry. -*/ -int lsmTreeCursorValid(TreeCursor *pCsr){ - return (pCsr && (pCsr->pSave || pCsr->iNode>=0)); -} - -/* -** Store a mark in *pMark. Later on, a call to lsmTreeRollback() with a -** pointer to the same TreeMark structure may be used to roll the tree -** contents back to their current state. -*/ -void lsmTreeMark(lsm_db *pDb, TreeMark *pMark){ - pMark->iRoot = pDb->treehdr.root.iRoot; - pMark->nHeight = pDb->treehdr.root.nHeight; - pMark->iWrite = pDb->treehdr.iWrite; - pMark->nChunk = pDb->treehdr.nChunk; - pMark->iNextShmid = pDb->treehdr.iNextShmid; - pMark->iRollback = intArraySize(&pDb->rollback); -} - -/* -** Roll back to mark pMark. Structure *pMark should have been previously -** populated by a call to lsmTreeMark(). -*/ -void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark){ - int iIdx; - int nIdx; - u32 iNext; - ShmChunk *pChunk; - u32 iChunk; - u32 iShmid; - - /* Revert all required v2 pointers. */ - nIdx = intArraySize(&pDb->rollback); - for(iIdx = pMark->iRollback; iIdx<nIdx; iIdx++){ - TreeNode *pNode; - pNode = treeShmptr(pDb, intArrayEntry(&pDb->rollback, iIdx)); - assert( pNode ); - pNode->iV2 = 0; - pNode->iV2Child = 0; - pNode->iV2Ptr = 0; - } - intArrayTruncate(&pDb->rollback, pMark->iRollback); - - /* Restore the free-chunk list. */ - assert( pMark->iWrite!=0 ); - iChunk = treeOffsetToChunk(pMark->iWrite-1); - pChunk = treeShmChunk(pDb, iChunk); - iNext = pChunk->iNext; - pChunk->iNext = 0; - - pChunk = treeShmChunk(pDb, pDb->treehdr.iFirst); - iShmid = pChunk->iShmid-1; - - while( iNext ){ - u32 iFree = iNext; /* Current chunk being rollback-freed */ - ShmChunk *pFree; /* Pointer to chunk iFree */ - - pFree = treeShmChunk(pDb, iFree); - iNext = pFree->iNext; - - if( iFree<pMark->nChunk ){ - pFree->iNext = pDb->treehdr.iFirst; - pFree->iShmid = iShmid--; - pDb->treehdr.iFirst = iFree; - } - } - - /* Restore the tree-header fields */ - pDb->treehdr.root.iRoot = pMark->iRoot; - pDb->treehdr.root.nHeight = pMark->nHeight; - pDb->treehdr.iWrite = pMark->iWrite; - pDb->treehdr.nChunk = pMark->nChunk; - pDb->treehdr.iNextShmid = pMark->iNextShmid; -} - -/* -** Load the in-memory tree header from shared-memory into pDb->treehdr. -** If the header cannot be loaded, return LSM_PROTOCOL. -** -** If the header is successfully loaded and parameter piRead is not NULL, -** is is set to 1 if the header was loaded from ShmHeader.hdr1, or 2 if -** the header was loaded from ShmHeader.hdr2. -*/ -int lsmTreeLoadHeader(lsm_db *pDb, int *piRead){ - int nRem = LSM_ATTEMPTS_BEFORE_PROTOCOL; - while( (nRem--)>0 ){ - ShmHeader *pShm = pDb->pShmhdr; - - memcpy(&pDb->treehdr, &pShm->hdr1, sizeof(TreeHeader)); - if( treeHeaderChecksumOk(&pDb->treehdr) ){ - if( piRead ) *piRead = 1; - return LSM_OK; - } - memcpy(&pDb->treehdr, &pShm->hdr2, sizeof(TreeHeader)); - if( treeHeaderChecksumOk(&pDb->treehdr) ){ - if( piRead ) *piRead = 2; - return LSM_OK; - } - - lsmShmBarrier(pDb); - } - return LSM_PROTOCOL_BKPT; -} - -int lsmTreeLoadHeaderOk(lsm_db *pDb, int iRead){ - TreeHeader *p = (iRead==1) ? &pDb->pShmhdr->hdr1 : &pDb->pShmhdr->hdr2; - assert( iRead==1 || iRead==2 ); - return (0==memcmp(pDb->treehdr.aCksum, p->aCksum, sizeof(u32)*2)); -} - -/* -** This function is called to conclude a transaction. If argument bCommit -** is true, the transaction is committed. Otherwise it is rolled back. -*/ -int lsmTreeEndTransaction(lsm_db *pDb, int bCommit){ - ShmHeader *pShm = pDb->pShmhdr; - - treeHeaderChecksum(&pDb->treehdr, pDb->treehdr.aCksum); - memcpy(&pShm->hdr2, &pDb->treehdr, sizeof(TreeHeader)); - lsmShmBarrier(pDb); - memcpy(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)); - pShm->bWriter = 0; - intArrayFree(pDb->pEnv, &pDb->rollback); - - return LSM_OK; -} - -#ifndef NDEBUG -static int assert_delete_ranges_match(lsm_db *db){ - int prev = 0; - TreeBlob blob = {0, 0}; - TreeCursor csr; /* Cursor used to iterate through tree */ - int rc; - - treeCursorInit(db, 0, &csr); - for( rc = lsmTreeCursorEnd(&csr, 0); - rc==LSM_OK && lsmTreeCursorValid(&csr); - rc = lsmTreeCursorNext(&csr) - ){ - TreeKey *pKey = csrGetKey(&csr, &blob, &rc); - if( rc!=LSM_OK ) break; - assert( ((prev&LSM_START_DELETE)==0)==((pKey->flags&LSM_END_DELETE)==0) ); - prev = pKey->flags; - } - - tblobFree(csr.pDb, &csr.blob); - tblobFree(csr.pDb, &blob); - - return 1; -} - -static int treeCountEntries(lsm_db *db){ - TreeCursor csr; /* Cursor used to iterate through tree */ - int rc; - int nEntry = 0; - - treeCursorInit(db, 0, &csr); - for( rc = lsmTreeCursorEnd(&csr, 0); - rc==LSM_OK && lsmTreeCursorValid(&csr); - rc = lsmTreeCursorNext(&csr) - ){ - nEntry++; - } - - tblobFree(csr.pDb, &csr.blob); - - return nEntry; -} -#endif diff --git a/ext/lsm1/lsm_unix.c b/ext/lsm1/lsm_unix.c deleted file mode 100644 index 88952d15f..000000000 --- a/ext/lsm1/lsm_unix.c +++ /dev/null @@ -1,753 +0,0 @@ -/* -** 2011-12-03 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Unix-specific run-time environment implementation for LSM. -*/ - -#ifndef _WIN32 - -#if defined(__GNUC__) || defined(__TINYC__) -/* workaround for ftruncate() visibility on gcc. */ -# ifndef _XOPEN_SOURCE -# define _XOPEN_SOURCE 500 -# endif -#endif - -#include <unistd.h> -#include <sys/types.h> - -#include <sys/stat.h> -#include <fcntl.h> -#include <assert.h> -#include <string.h> - -#include <stdlib.h> -#include <stdarg.h> -#include <stdio.h> -#include <ctype.h> - -#include <unistd.h> -#include <errno.h> - -#include <sys/mman.h> -#include "lsmInt.h" - -/* There is no fdatasync() call on Android */ -#ifdef __ANDROID__ -# define fdatasync(x) fsync(x) -#endif - -/* -** An open file is an instance of the following object -*/ -typedef struct PosixFile PosixFile; -struct PosixFile { - lsm_env *pEnv; /* The run-time environment */ - const char *zName; /* Full path to file */ - int fd; /* The open file descriptor */ - int shmfd; /* Shared memory file-descriptor */ - void *pMap; /* Pointer to mapping of file fd */ - off_t nMap; /* Size of mapping at pMap in bytes */ - int nShm; /* Number of entries in array apShm[] */ - void **apShm; /* Array of 32K shared memory segments */ -}; - -static char *posixShmFile(PosixFile *p){ - char *zShm; - int nName = strlen(p->zName); - zShm = (char *)lsmMalloc(p->pEnv, nName+4+1); - if( zShm ){ - memcpy(zShm, p->zName, nName); - memcpy(&zShm[nName], "-shm", 5); - } - return zShm; -} - -static int lsmPosixOsOpen( - lsm_env *pEnv, - const char *zFile, - int flags, - lsm_file **ppFile -){ - int rc = LSM_OK; - PosixFile *p; - - p = lsm_malloc(pEnv, sizeof(PosixFile)); - if( p==0 ){ - rc = LSM_NOMEM; - }else{ - int bReadonly = (flags & LSM_OPEN_READONLY); - int oflags = (bReadonly ? O_RDONLY : (O_RDWR|O_CREAT)); - memset(p, 0, sizeof(PosixFile)); - p->zName = zFile; - p->pEnv = pEnv; - p->fd = open(zFile, oflags, 0644); - if( p->fd<0 ){ - lsm_free(pEnv, p); - p = 0; - if( errno==ENOENT ){ - rc = lsmErrorBkpt(LSM_IOERR_NOENT); - }else{ - rc = LSM_IOERR_BKPT; - } - } - } - - *ppFile = (lsm_file *)p; - return rc; -} - -static int lsmPosixOsWrite( - lsm_file *pFile, /* File to write to */ - lsm_i64 iOff, /* Offset to write to */ - void *pData, /* Write data from this buffer */ - int nData /* Bytes of data to write */ -){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - off_t offset; - - offset = lseek(p->fd, (off_t)iOff, SEEK_SET); - if( offset!=iOff ){ - rc = LSM_IOERR_BKPT; - }else{ - ssize_t prc = write(p->fd, pData, (size_t)nData); - if( prc<0 ) rc = LSM_IOERR_BKPT; - } - - return rc; -} - -static int lsmPosixOsTruncate( - lsm_file *pFile, /* File to write to */ - lsm_i64 nSize /* Size to truncate file to */ -){ - PosixFile *p = (PosixFile *)pFile; - int rc = LSM_OK; /* Return code */ - int prc; /* Posix Return Code */ - struct stat sStat; /* Result of fstat() invocation */ - - prc = fstat(p->fd, &sStat); - if( prc==0 && sStat.st_size>nSize ){ - prc = ftruncate(p->fd, (off_t)nSize); - } - if( prc<0 ) rc = LSM_IOERR_BKPT; - - return rc; -} - -static int lsmPosixOsRead( - lsm_file *pFile, /* File to read from */ - lsm_i64 iOff, /* Offset to read from */ - void *pData, /* Read data into this buffer */ - int nData /* Bytes of data to read */ -){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - off_t offset; - - offset = lseek(p->fd, (off_t)iOff, SEEK_SET); - if( offset!=iOff ){ - rc = LSM_IOERR_BKPT; - }else{ - ssize_t prc = read(p->fd, pData, (size_t)nData); - if( prc<0 ){ - rc = LSM_IOERR_BKPT; - }else if( prc<nData ){ - memset(&((u8 *)pData)[prc], 0, nData - prc); - } - - } - - return rc; -} - -static int lsmPosixOsSync(lsm_file *pFile){ - int rc = LSM_OK; - -#ifndef LSM_NO_SYNC - PosixFile *p = (PosixFile *)pFile; - int prc = 0; - - if( p->pMap ){ - prc = msync(p->pMap, p->nMap, MS_SYNC); - } - if( prc==0 ) prc = fdatasync(p->fd); - if( prc<0 ) rc = LSM_IOERR_BKPT; -#else - (void)pFile; -#endif - - return rc; -} - -static int lsmPosixOsSectorSize(lsm_file *pFile){ - return 512; -} - -static int lsmPosixOsRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - off_t iSz; - int prc; - PosixFile *p = (PosixFile *)pFile; - struct stat buf; - - /* If the file is between 0 and 2MB in size, extend it in chunks of 256K. - ** Thereafter, in chunks of 1MB at a time. */ - const int aIncrSz[] = {256*1024, 1024*1024}; - int nIncrSz = aIncrSz[iMin>(2*1024*1024)]; - - if( p->pMap ){ - munmap(p->pMap, p->nMap); - *ppOut = p->pMap = 0; - *pnOut = p->nMap = 0; - } - - if( iMin>=0 ){ - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - iSz = buf.st_size; - if( iSz<iMin ){ - iSz = ((iMin + nIncrSz-1) / nIncrSz) * nIncrSz; - prc = ftruncate(p->fd, iSz); - if( prc!=0 ) return LSM_IOERR_BKPT; - } - - p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); - if( p->pMap==MAP_FAILED ){ - p->pMap = 0; - return LSM_IOERR_BKPT; - } - p->nMap = iSz; - } - - *ppOut = p->pMap; - *pnOut = p->nMap; - return LSM_OK; -} - -static int lsmPosixOsFullpath( - lsm_env *pEnv, - const char *zName, - char *zOut, - int *pnOut -){ - int nBuf = *pnOut; - int nReq; - - if( zName[0]!='/' ){ - char *z; - char *zTmp; - int nTmp = 512; - zTmp = lsmMalloc(pEnv, nTmp); - while( zTmp ){ - z = getcwd(zTmp, nTmp); - if( z || errno!=ERANGE ) break; - nTmp = nTmp*2; - zTmp = lsmReallocOrFree(pEnv, zTmp, nTmp); - } - if( zTmp==0 ) return LSM_NOMEM_BKPT; - if( z==0 ) return LSM_IOERR_BKPT; - assert( z==zTmp ); - - nTmp = strlen(zTmp); - nReq = nTmp + 1 + strlen(zName) + 1; - if( nReq<=nBuf ){ - memcpy(zOut, zTmp, nTmp); - zOut[nTmp] = '/'; - memcpy(&zOut[nTmp+1], zName, strlen(zName)+1); - } - lsmFree(pEnv, zTmp); - }else{ - nReq = strlen(zName)+1; - if( nReq<=nBuf ){ - memcpy(zOut, zName, strlen(zName)+1); - } - } - - *pnOut = nReq; - return LSM_OK; -} - -static int lsmPosixOsFileid( - lsm_file *pFile, - void *pBuf, - int *pnBuf -){ - int prc; - int nBuf; - int nReq; - PosixFile *p = (PosixFile *)pFile; - struct stat buf; - - nBuf = *pnBuf; - nReq = (sizeof(buf.st_dev) + sizeof(buf.st_ino)); - *pnBuf = nReq; - if( nReq>nBuf ) return LSM_OK; - - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - - memcpy(pBuf, &buf.st_dev, sizeof(buf.st_dev)); - memcpy(&(((u8 *)pBuf)[sizeof(buf.st_dev)]), &buf.st_ino, sizeof(buf.st_ino)); - return LSM_OK; -} - -static int lsmPosixOsUnlink(lsm_env *pEnv, const char *zFile){ - int prc = unlink(zFile); - return prc ? LSM_IOERR_BKPT : LSM_OK; -} - -static int lsmPosixOsLock(lsm_file *pFile, int iLock, int eType){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - static const short aType[3] = { F_UNLCK, F_RDLCK, F_WRLCK }; - struct flock lock; - - assert( aType[LSM_LOCK_UNLOCK]==F_UNLCK ); - assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); - assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); - assert( eType>=0 && eType<array_size(aType) ); - assert( iLock>0 && iLock<=32 ); - - memset(&lock, 0, sizeof(lock)); - lock.l_whence = SEEK_SET; - lock.l_len = 1; - lock.l_type = aType[eType]; - lock.l_start = (4096-iLock); - - if( fcntl(p->fd, F_SETLK, &lock) ){ - int e = errno; - if( e==EACCES || e==EAGAIN ){ - rc = LSM_BUSY; - }else{ - rc = LSM_IOERR_BKPT; - } - } - - return rc; -} - -static int lsmPosixOsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - static const short aType[3] = { 0, F_RDLCK, F_WRLCK }; - struct flock lock; - - assert( eType==LSM_LOCK_SHARED || eType==LSM_LOCK_EXCL ); - assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); - assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); - assert( eType>=0 && eType<array_size(aType) ); - assert( iLock>0 && iLock<=32 ); - - memset(&lock, 0, sizeof(lock)); - lock.l_whence = SEEK_SET; - lock.l_len = nLock; - lock.l_type = aType[eType]; - lock.l_start = (4096-iLock-nLock+1); - - if( fcntl(p->fd, F_GETLK, &lock) ){ - rc = LSM_IOERR_BKPT; - }else if( lock.l_type!=F_UNLCK ){ - rc = LSM_BUSY; - } - - return rc; -} - -static int lsmPosixOsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ - PosixFile *p = (PosixFile *)pFile; - - *ppShm = 0; - assert( sz==LSM_SHM_CHUNK_SIZE ); - if( iChunk>=p->nShm ){ - int i; - void **apNew; - int nNew = iChunk+1; - off_t nReq = nNew * LSM_SHM_CHUNK_SIZE; - struct stat sStat; - - /* If the shared-memory file has not been opened, open it now. */ - if( p->shmfd<=0 ){ - char *zShm = posixShmFile(p); - if( !zShm ) return LSM_NOMEM_BKPT; - p->shmfd = open(zShm, O_RDWR|O_CREAT, 0644); - lsmFree(p->pEnv, zShm); - if( p->shmfd<0 ){ - return LSM_IOERR_BKPT; - } - } - - /* If the shared-memory file is not large enough to contain the - ** requested chunk, cause it to grow. */ - if( fstat(p->shmfd, &sStat) ){ - return LSM_IOERR_BKPT; - } - if( sStat.st_size<nReq ){ - if( ftruncate(p->shmfd, nReq) ){ - return LSM_IOERR_BKPT; - } - } - - apNew = (void **)lsmRealloc(p->pEnv, p->apShm, sizeof(void *) * nNew); - if( !apNew ) return LSM_NOMEM_BKPT; - for(i=p->nShm; i<nNew; i++){ - apNew[i] = 0; - } - p->apShm = apNew; - p->nShm = nNew; - } - - if( p->apShm[iChunk]==0 ){ - p->apShm[iChunk] = mmap(0, LSM_SHM_CHUNK_SIZE, - PROT_READ|PROT_WRITE, MAP_SHARED, p->shmfd, iChunk*LSM_SHM_CHUNK_SIZE - ); - if( p->apShm[iChunk]==MAP_FAILED ){ - p->apShm[iChunk] = 0; - return LSM_IOERR_BKPT; - } - } - - *ppShm = p->apShm[iChunk]; - return LSM_OK; -} - -static void lsmPosixOsShmBarrier(void){ -} - -static int lsmPosixOsShmUnmap(lsm_file *pFile, int bDelete){ - PosixFile *p = (PosixFile *)pFile; - if( p->shmfd>0 ){ - int i; - for(i=0; i<p->nShm; i++){ - if( p->apShm[i] ){ - munmap(p->apShm[i], LSM_SHM_CHUNK_SIZE); - p->apShm[i] = 0; - } - } - close(p->shmfd); - p->shmfd = 0; - if( bDelete ){ - char *zShm = posixShmFile(p); - if( zShm ) unlink(zShm); - lsmFree(p->pEnv, zShm); - } - } - return LSM_OK; -} - - -static int lsmPosixOsClose(lsm_file *pFile){ - PosixFile *p = (PosixFile *)pFile; - lsmPosixOsShmUnmap(pFile, 0); - if( p->pMap ) munmap(p->pMap, p->nMap); - close(p->fd); - lsm_free(p->pEnv, p->apShm); - lsm_free(p->pEnv, p); - return LSM_OK; -} - -static int lsmPosixOsSleep(lsm_env *pEnv, int us){ -#if 0 - /* Apparently on Android usleep() returns void */ - if( usleep(us) ) return LSM_IOERR; -#endif - usleep(us); - return LSM_OK; -} - -/**************************************************************************** -** Memory allocation routines. -*/ -#define BLOCK_HDR_SIZE ROUND8( sizeof(size_t) ) - -static void *lsmPosixOsMalloc(lsm_env *pEnv, size_t N){ - unsigned char * m; - N += BLOCK_HDR_SIZE; - m = (unsigned char *)malloc(N); - *((size_t*)m) = N; - return m + BLOCK_HDR_SIZE; -} - -static void lsmPosixOsFree(lsm_env *pEnv, void *p){ - if(p){ - free( ((unsigned char *)p) - BLOCK_HDR_SIZE ); - } -} - -static void *lsmPosixOsRealloc(lsm_env *pEnv, void *p, size_t N){ - unsigned char * m = (unsigned char *)p; - if(1>N){ - lsmPosixOsFree( pEnv, p ); - return NULL; - }else if(NULL==p){ - return lsmPosixOsMalloc(pEnv, N); - }else{ - void * re = NULL; - m -= BLOCK_HDR_SIZE; -#if 0 /* arguable: don't shrink */ - size_t * sz = (size_t*)m; - if(*sz >= (size_t)N){ - return p; - } -#endif - re = realloc( m, N + BLOCK_HDR_SIZE ); - if(re){ - m = (unsigned char *)re; - *((size_t*)m) = N; - return m + BLOCK_HDR_SIZE; - }else{ - return NULL; - } - } -} - -static size_t lsmPosixOsMSize(lsm_env *pEnv, void *p){ - unsigned char * m = (unsigned char *)p; - return *((size_t*)(m-BLOCK_HDR_SIZE)); -} -#undef BLOCK_HDR_SIZE - - -#ifdef LSM_MUTEX_PTHREADS -/************************************************************************* -** Mutex methods for pthreads based systems. If LSM_MUTEX_PTHREADS is -** missing then a no-op implementation of mutexes found in lsm_mutex.c -** will be used instead. -*/ -#include <pthread.h> - -typedef struct PthreadMutex PthreadMutex; -struct PthreadMutex { - lsm_env *pEnv; - pthread_mutex_t mutex; -#ifdef LSM_DEBUG - pthread_t owner; -#endif -}; - -#ifdef LSM_DEBUG -# define LSM_PTHREAD_STATIC_MUTEX { 0, PTHREAD_MUTEX_INITIALIZER, 0 } -#else -# define LSM_PTHREAD_STATIC_MUTEX { 0, PTHREAD_MUTEX_INITIALIZER } -#endif - -static int lsmPosixOsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - static PthreadMutex sMutex[2] = { - LSM_PTHREAD_STATIC_MUTEX, - LSM_PTHREAD_STATIC_MUTEX - }; - - assert( iMutex==LSM_MUTEX_GLOBAL || iMutex==LSM_MUTEX_HEAP ); - assert( LSM_MUTEX_GLOBAL==1 && LSM_MUTEX_HEAP==2 ); - - *ppStatic = (lsm_mutex *)&sMutex[iMutex-1]; - return LSM_OK; -} - -static int lsmPosixOsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - PthreadMutex *pMutex; /* Pointer to new mutex */ - pthread_mutexattr_t attr; /* Attributes object */ - - pMutex = (PthreadMutex *)lsmMallocZero(pEnv, sizeof(PthreadMutex)); - if( !pMutex ) return LSM_NOMEM_BKPT; - - pMutex->pEnv = pEnv; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&pMutex->mutex, &attr); - pthread_mutexattr_destroy(&attr); - - *ppNew = (lsm_mutex *)pMutex; - return LSM_OK; -} - -static void lsmPosixOsMutexDel(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - pthread_mutex_destroy(&pMutex->mutex); - lsmFree(pMutex->pEnv, pMutex); -} - -static void lsmPosixOsMutexEnter(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - pthread_mutex_lock(&pMutex->mutex); - -#ifdef LSM_DEBUG - assert( !pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = pthread_self(); - assert( pthread_equal(pMutex->owner, pthread_self()) ); -#endif -} - -static int lsmPosixOsMutexTry(lsm_mutex *p){ - int ret; - PthreadMutex *pMutex = (PthreadMutex *)p; - ret = pthread_mutex_trylock(&pMutex->mutex); -#ifdef LSM_DEBUG - if( ret==0 ){ - assert( !pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = pthread_self(); - assert( pthread_equal(pMutex->owner, pthread_self()) ); - } -#endif - return ret; -} - -static void lsmPosixOsMutexLeave(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; -#ifdef LSM_DEBUG - assert( pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = 0; - assert( !pthread_equal(pMutex->owner, pthread_self()) ); -#endif - pthread_mutex_unlock(&pMutex->mutex); -} - -#ifdef LSM_DEBUG -static int lsmPosixOsMutexHeld(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - return pMutex ? pthread_equal(pMutex->owner, pthread_self()) : 1; -} -static int lsmPosixOsMutexNotHeld(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - return pMutex ? !pthread_equal(pMutex->owner, pthread_self()) : 1; -} -#endif -/* -** End of pthreads mutex implementation. -*************************************************************************/ -#else -/************************************************************************* -** Noop mutex implementation -*/ -typedef struct NoopMutex NoopMutex; -struct NoopMutex { - lsm_env *pEnv; /* Environment handle (for xFree()) */ - int bHeld; /* True if mutex is held */ - int bStatic; /* True for a static mutex */ -}; -static NoopMutex aStaticNoopMutex[2] = { - {0, 0, 1}, - {0, 0, 1}, -}; - -static int lsmPosixOsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - assert( iMutex>=1 && iMutex<=(int)array_size(aStaticNoopMutex) ); - *ppStatic = (lsm_mutex *)&aStaticNoopMutex[iMutex-1]; - return LSM_OK; -} -static int lsmPosixOsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - NoopMutex *p; - p = (NoopMutex *)lsmMallocZero(pEnv, sizeof(NoopMutex)); - if( p ) p->pEnv = pEnv; - *ppNew = (lsm_mutex *)p; - return (p ? LSM_OK : LSM_NOMEM_BKPT); -} -static void lsmPosixOsMutexDel(lsm_mutex *pMutex) { - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bStatic==0 && p->pEnv ); - lsmFree(p->pEnv, p); -} -static void lsmPosixOsMutexEnter(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; -} -static int lsmPosixOsMutexTry(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; - return 0; -} -static void lsmPosixOsMutexLeave(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==1 ); - p->bHeld = 0; -} -#ifdef LSM_DEBUG -static int lsmPosixOsMutexHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? p->bHeld : 1; -} -static int lsmPosixOsMutexNotHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? !p->bHeld : 1; -} -#endif -/***************************************************************************/ -#endif /* else LSM_MUTEX_NONE */ - -/* Without LSM_DEBUG, the MutexHeld tests are never called */ -#ifndef LSM_DEBUG -# define lsmPosixOsMutexHeld 0 -# define lsmPosixOsMutexNotHeld 0 -#endif - -lsm_env *lsm_default_env(void){ - static lsm_env posix_env = { - sizeof(lsm_env), /* nByte */ - 1, /* iVersion */ - /***** file i/o ******************/ - 0, /* pVfsCtx */ - lsmPosixOsFullpath, /* xFullpath */ - lsmPosixOsOpen, /* xOpen */ - lsmPosixOsRead, /* xRead */ - lsmPosixOsWrite, /* xWrite */ - lsmPosixOsTruncate, /* xTruncate */ - lsmPosixOsSync, /* xSync */ - lsmPosixOsSectorSize, /* xSectorSize */ - lsmPosixOsRemap, /* xRemap */ - lsmPosixOsFileid, /* xFileid */ - lsmPosixOsClose, /* xClose */ - lsmPosixOsUnlink, /* xUnlink */ - lsmPosixOsLock, /* xLock */ - lsmPosixOsTestLock, /* xTestLock */ - lsmPosixOsShmMap, /* xShmMap */ - lsmPosixOsShmBarrier, /* xShmBarrier */ - lsmPosixOsShmUnmap, /* xShmUnmap */ - /***** memory allocation *********/ - 0, /* pMemCtx */ - lsmPosixOsMalloc, /* xMalloc */ - lsmPosixOsRealloc, /* xRealloc */ - lsmPosixOsFree, /* xFree */ - lsmPosixOsMSize, /* xSize */ - /***** mutexes *********************/ - 0, /* pMutexCtx */ - lsmPosixOsMutexStatic, /* xMutexStatic */ - lsmPosixOsMutexNew, /* xMutexNew */ - lsmPosixOsMutexDel, /* xMutexDel */ - lsmPosixOsMutexEnter, /* xMutexEnter */ - lsmPosixOsMutexTry, /* xMutexTry */ - lsmPosixOsMutexLeave, /* xMutexLeave */ - lsmPosixOsMutexHeld, /* xMutexHeld */ - lsmPosixOsMutexNotHeld, /* xMutexNotHeld */ - /***** other *********************/ - lsmPosixOsSleep, /* xSleep */ - }; - return &posix_env; -} - -#endif diff --git a/ext/lsm1/lsm_varint.c b/ext/lsm1/lsm_varint.c deleted file mode 100644 index f690e3b06..000000000 --- a/ext/lsm1/lsm_varint.c +++ /dev/null @@ -1,201 +0,0 @@ - -/* -** 2012-02-08 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** SQLite4-compatible varint implementation. -*/ -#include "lsmInt.h" - -/************************************************************************* -** The following is a copy of the varint.c module from SQLite 4. -*/ - -/* -** Decode the varint in z[]. Write the integer value into *pResult and -** return the number of bytes in the varint. -*/ -static int lsmSqlite4GetVarint64(const unsigned char *z, u64 *pResult){ - unsigned int x; - if( z[0]<=240 ){ - *pResult = z[0]; - return 1; - } - if( z[0]<=248 ){ - *pResult = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( z[0]==249 ){ - *pResult = 2288 + 256*z[1] + z[2]; - return 3; - } - if( z[0]==250 ){ - *pResult = (z[1]<<16) + (z[2]<<8) + z[3]; - return 4; - } - x = (z[1]<<24) + (z[2]<<16) + (z[3]<<8) + z[4]; - if( z[0]==251 ){ - *pResult = x; - return 5; - } - if( z[0]==252 ){ - *pResult = (((u64)x)<<8) + z[5]; - return 6; - } - if( z[0]==253 ){ - *pResult = (((u64)x)<<16) + (z[5]<<8) + z[6]; - return 7; - } - if( z[0]==254 ){ - *pResult = (((u64)x)<<24) + (z[5]<<16) + (z[6]<<8) + z[7]; - return 8; - } - *pResult = (((u64)x)<<32) + - (0xffffffff & ((z[5]<<24) + (z[6]<<16) + (z[7]<<8) + z[8])); - return 9; -} - -/* -** Write a 32-bit unsigned integer as 4 big-endian bytes. -*/ -static void lsmVarintWrite32(unsigned char *z, unsigned int y){ - z[0] = (unsigned char)(y>>24); - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); -} - -/* -** Write a varint into z[]. The buffer z[] must be at least 9 characters -** long to accommodate the largest possible varint. Return the number of -** bytes of z[] used. -*/ -static int lsmSqlite4PutVarint64(unsigned char *z, u64 x){ - unsigned int w, y; - if( x<=240 ){ - z[0] = (unsigned char)x; - return 1; - } - if( x<=2287 ){ - y = (unsigned int)(x - 240); - z[0] = (unsigned char)(y/256 + 241); - z[1] = (unsigned char)(y%256); - return 2; - } - if( x<=67823 ){ - y = (unsigned int)(x - 2288); - z[0] = 249; - z[1] = (unsigned char)(y/256); - z[2] = (unsigned char)(y%256); - return 3; - } - y = (unsigned int)x; - w = (unsigned int)(x>>32); - if( w==0 ){ - if( y<=16777215 ){ - z[0] = 250; - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); - return 4; - } - z[0] = 251; - lsmVarintWrite32(z+1, y); - return 5; - } - if( w<=255 ){ - z[0] = 252; - z[1] = (unsigned char)w; - lsmVarintWrite32(z+2, y); - return 6; - } - if( w<=32767 ){ - z[0] = 253; - z[1] = (unsigned char)(w>>8); - z[2] = (unsigned char)w; - lsmVarintWrite32(z+3, y); - return 7; - } - if( w<=16777215 ){ - z[0] = 254; - z[1] = (unsigned char)(w>>16); - z[2] = (unsigned char)(w>>8); - z[3] = (unsigned char)w; - lsmVarintWrite32(z+4, y); - return 8; - } - z[0] = 255; - lsmVarintWrite32(z+1, w); - lsmVarintWrite32(z+5, y); - return 9; -} - -/* -** End of SQLite 4 code. -*************************************************************************/ - -int lsmVarintPut64(u8 *aData, i64 iVal){ - return lsmSqlite4PutVarint64(aData, (u64)iVal); -} - -int lsmVarintGet64(const u8 *aData, i64 *piVal){ - return lsmSqlite4GetVarint64(aData, (u64 *)piVal); -} - -int lsmVarintPut32(u8 *aData, int iVal){ - return lsmSqlite4PutVarint64(aData, (u64)iVal); -} - -int lsmVarintGet32(u8 *z, int *piVal){ - u64 i; - int ret; - - if( z[0]<=240 ){ - *piVal = z[0]; - return 1; - } - if( z[0]<=248 ){ - *piVal = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( z[0]==249 ){ - *piVal = 2288 + 256*z[1] + z[2]; - return 3; - } - if( z[0]==250 ){ - *piVal = (z[1]<<16) + (z[2]<<8) + z[3]; - return 4; - } - - ret = lsmSqlite4GetVarint64(z, &i); - *piVal = (int)i; - return ret; -} - -int lsmVarintLen32(int n){ - u8 aData[9]; - return lsmVarintPut32(aData, n); -} - -int lsmVarintLen64(i64 n){ - u8 aData[9]; - return lsmVarintPut64(aData, n); -} - -/* -** The argument is the first byte of a varint. This function returns the -** total number of bytes in the entire varint (including the first byte). -*/ -int lsmVarintSize(u8 c){ - if( c<241 ) return 1; - if( c<249 ) return 2; - return (int)(c - 246); -} diff --git a/ext/lsm1/lsm_vtab.c b/ext/lsm1/lsm_vtab.c deleted file mode 100644 index 8c21923e1..000000000 --- a/ext/lsm1/lsm_vtab.c +++ /dev/null @@ -1,1084 +0,0 @@ -/* -** 2015-11-16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements a virtual table for SQLite3 around the LSM -** storage engine from SQLite4. -** -** USAGE -** -** CREATE VIRTUAL TABLE demo USING lsm1(filename,key,keytype,value1,...); -** -** The filename parameter is the name of the LSM database file, which is -** separate and distinct from the SQLite3 database file. -** -** The keytype must be one of: UINT, TEXT, BLOB. All keys must be of that -** one type. "UINT" means unsigned integer. The values may be of any -** SQLite datatype: BLOB, TEXT, INTEGER, FLOAT, or NULL. -** -** The virtual table contains read-only hidden columns: -** -** lsm1_key A BLOB which is the raw LSM key. If the "keytype" -** is BLOB or TEXT then this column is exactly the -** same as the key. For the UINT keytype, this column -** will be a variable-length integer encoding of the key. -** -** lsm1_value A BLOB which is the raw LSM value. All of the value -** columns are packed into this BLOB using the encoding -** described below. -** -** Attempts to write values into the lsm1_key and lsm1_value columns are -** silently ignored. -** -** EXAMPLE -** -** The virtual table declared this way: -** -** CREATE VIRTUAL TABLE demo2 USING lsm1('x.lsm',id,UINT,a,b,c,d); -** -** Results in a new virtual table named "demo2" that acts as if it has -** the following schema: -** -** CREATE TABLE demo2( -** id UINT PRIMARY KEY ON CONFLICT REPLACE, -** a ANY, -** b ANY, -** c ANY, -** d ANY, -** lsm1_key BLOB HIDDEN, -** lsm1_value BLOB HIDDEN -** ) WITHOUT ROWID; -** -** -** -** INTERNALS -** -** The key encoding for BLOB and TEXT is just a copy of the blob or text. -** UTF-8 is used for text. The key encoding for UINT is the variable-length -** integer format at https://sqlite.org/src4/doc/trunk/www/varint.wiki. -** -** The values are encoded as a single blob (since that is what lsm stores as -** its content). There is a "type integer" followed by "content" for each -** value, alternating back and forth. The content might be empty. -** -** TYPE1 CONTENT1 TYPE2 CONTENT2 TYPE3 CONTENT3 .... -** -** Each "type integer" is encoded as a variable-length integer in the -** format of the link above. Let the type integer be T. The actual -** datatype is an integer 0-5 equal to T%6. Values 1 through 5 correspond -** to SQLITE_INTEGER through SQLITE_NULL. The size of the content in bytes -** is T/6. Type value 0 means that the value is an integer whose actual -** values is T/6 and there is no content. The type-value-0 integer format -** only works for integers in the range of 0 through 40. -** -** There is no content for NULL or type-0 integers. For BLOB and TEXT -** values, the content is the blob data or the UTF-8 text data. For -** non-negative integers X, the content is a variable-length integer X*2. -** For negative integers Y, the content is varaible-length integer (1-Y)*2+1. -** For FLOAT values, the content is the IEEE754 floating point value in -** native byte-order. This means that FLOAT values will be corrupted when -** database file is moved between big-endian and little-endian machines. -*/ -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 -#include "lsm.h" -#include <assert.h> -#include <string.h> - -/* Forward declaration of subclasses of virtual table objects */ -typedef struct lsm1_vtab lsm1_vtab; -typedef struct lsm1_cursor lsm1_cursor; -typedef struct lsm1_vblob lsm1_vblob; - -/* Primitive types */ -typedef unsigned char u8; -typedef unsigned int u32; -typedef sqlite3_uint64 u64; - -/* An open connection to an LSM table */ -struct lsm1_vtab { - sqlite3_vtab base; /* Base class - must be first */ - lsm_db *pDb; /* Open connection to the LSM table */ - u8 keyType; /* SQLITE_BLOB, _TEXT, or _INTEGER */ - u32 nVal; /* Number of value columns */ -}; - - -/* lsm1_cursor is a subclass of sqlite3_vtab_cursor which will -** serve as the underlying representation of a cursor that scans -** over rows of the result -*/ -struct lsm1_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - lsm_cursor *pLsmCur; /* The LSM cursor */ - u8 isDesc; /* 0: scan forward. 1: scan reverse */ - u8 atEof; /* True if the scan is complete */ - u8 bUnique; /* True if no more than one row of output */ - u8 *zData; /* Content of the current row */ - u32 nData; /* Number of bytes in the current row */ - u8 *aeType; /* Types for all column values */ - u32 *aiOfst; /* Offsets to the various fields */ - u32 *aiLen; /* Length of each field */ - u8 *pKey2; /* Loop termination key, or NULL */ - u32 nKey2; /* Length of the loop termination key */ -}; - -/* An extensible buffer object. -** -** Content can be appended. Space to hold new content is automatically -** allocated. -*/ -struct lsm1_vblob { - u8 *a; /* Space to hold content, from sqlite3_malloc64() */ - u64 n; /* Bytes of space used */ - u64 nAlloc; /* Bytes of space allocated */ - u8 errNoMem; /* True if a memory allocation error has been seen */ -}; - -#if defined(__GNUC__) -# define LSM1_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) && _MSC_VER>=1310 -# define LSM1_NOINLINE __declspec(noinline) -#else -# define LSM1_NOINLINE -#endif - - -/* Increase the available space in the vblob object so that it can hold -** at least N more bytes. Return the number of errors. -*/ -static int lsm1VblobEnlarge(lsm1_vblob *p, u32 N){ - if( p->n+N>p->nAlloc ){ - if( p->errNoMem ) return 1; - p->nAlloc += N + (p->nAlloc ? p->nAlloc : N); - p->a = sqlite3_realloc64(p->a, p->nAlloc); - if( p->a==0 ){ - p->n = 0; - p->nAlloc = 0; - p->errNoMem = 1; - return 1; - } - p->nAlloc = sqlite3_msize(p->a); - } - return 0; -} - -/* Append N bytes to a vblob after first enlarging it */ -static LSM1_NOINLINE void lsm1VblobEnlargeAndAppend( - lsm1_vblob *p, - const u8 *pData, - u32 N -){ - if( p->n+N>p->nAlloc && lsm1VblobEnlarge(p, N) ) return; - memcpy(p->a+p->n, pData, N); - p->n += N; -} - -/* Append N bytes to a vblob */ -static void lsm1VblobAppend(lsm1_vblob *p, const u8 *pData, u32 N){ - sqlite3_int64 n = p->n; - if( n+N>p->nAlloc ){ - lsm1VblobEnlargeAndAppend(p, pData, N); - }else{ - p->n += N; - memcpy(p->a+n, pData, N); - } -} - -/* append text to a vblob */ -static void lsm1VblobAppendText(lsm1_vblob *p, const char *z){ - lsm1VblobAppend(p, (u8*)z, (u32)strlen(z)); -} - -/* Dequote the string */ -static void lsm1Dequote(char *z){ - int j; - char cQuote = z[0]; - size_t i, n; - - if( cQuote!='\'' && cQuote!='"' ) return; - n = strlen(z); - if( n<2 || z[n-1]!=z[0] ) return; - for(i=1, j=0; i<n-1; i++){ - if( z[i]==cQuote && z[i+1]==cQuote ) i++; - z[j++] = z[i]; - } - z[j] = 0; -} - - -/* -** The lsm1Connect() method is invoked to create a new -** lsm1_vtab that describes the virtual table. -*/ -static int lsm1Connect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - lsm1_vtab *pNew; - int rc; - char *zFilename; - u8 keyType = 0; - int i; - lsm1_vblob sql; - static const char *azTypes[] = { "UINT", "TEXT", "BLOB" }; - static const u8 aeTypes[] = { SQLITE_INTEGER, SQLITE_TEXT, SQLITE_BLOB }; - static const char *azArgName[] = {"filename", "key", "key type", "value1" }; - - for(i=0; i<sizeof(azArgName)/sizeof(azArgName[0]); i++){ - if( argc<i+4 || argv[i+3]==0 || argv[i+3][0]==0 ){ - *pzErr = sqlite3_mprintf("%s (%r) argument missing", - azArgName[i], i+1); - return SQLITE_ERROR; - } - } - for(i=0; i<sizeof(azTypes)/sizeof(azTypes[0]); i++){ - if( sqlite3_stricmp(azTypes[i],argv[5])==0 ){ - keyType = aeTypes[i]; - break; - } - } - if( keyType==0 ){ - *pzErr = sqlite3_mprintf("key type should be INT, TEXT, or BLOB"); - return SQLITE_ERROR; - } - *ppVtab = sqlite3_malloc( sizeof(*pNew) ); - pNew = (lsm1_vtab*)*ppVtab; - if( pNew==0 ){ - return SQLITE_NOMEM; - } - memset(pNew, 0, sizeof(*pNew)); - pNew->keyType = keyType; - rc = lsm_new(0, &pNew->pDb); - if( rc ){ - *pzErr = sqlite3_mprintf("lsm_new failed with error code %d", rc); - rc = SQLITE_ERROR; - goto connect_failed; - } - zFilename = sqlite3_mprintf("%s", argv[3]); - lsm1Dequote(zFilename); - rc = lsm_open(pNew->pDb, zFilename); - sqlite3_free(zFilename); - if( rc ){ - *pzErr = sqlite3_mprintf("lsm_open failed with %d", rc); - rc = SQLITE_ERROR; - goto connect_failed; - } - - memset(&sql, 0, sizeof(sql)); - lsm1VblobAppendText(&sql, "CREATE TABLE x("); - lsm1VblobAppendText(&sql, argv[4]); - lsm1VblobAppendText(&sql, " "); - lsm1VblobAppendText(&sql, argv[5]); - lsm1VblobAppendText(&sql, " PRIMARY KEY"); - for(i=6; i<argc; i++){ - lsm1VblobAppendText(&sql, ", "); - lsm1VblobAppendText(&sql, argv[i]); - pNew->nVal++; - } - lsm1VblobAppendText(&sql, - ", lsm1_command HIDDEN" - ", lsm1_key HIDDEN" - ", lsm1_value HIDDEN) WITHOUT ROWID"); - lsm1VblobAppend(&sql, (u8*)"", 1); - if( sql.errNoMem ){ - rc = SQLITE_NOMEM; - goto connect_failed; - } - rc = sqlite3_declare_vtab(db, (const char*)sql.a); - sqlite3_free(sql.a); - -connect_failed: - if( rc!=SQLITE_OK ){ - if( pNew ){ - if( pNew->pDb ) lsm_close(pNew->pDb); - sqlite3_free(pNew); - } - *ppVtab = 0; - } - return rc; -} - -/* -** This method is the destructor for lsm1_cursor objects. -*/ -static int lsm1Disconnect(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - lsm_close(p->pDb); - sqlite3_free(p); - return SQLITE_OK; -} - -/* -** Constructor for a new lsm1_cursor object. -*/ -static int lsm1Open(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - lsm1_cursor *pCur; - int rc; - pCur = sqlite3_malloc64( sizeof(*pCur) - + p->nVal*(sizeof(pCur->aiOfst)+sizeof(pCur->aiLen)+1) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - pCur->aiOfst = (u32*)&pCur[1]; - pCur->aiLen = &pCur->aiOfst[p->nVal]; - pCur->aeType = (u8*)&pCur->aiLen[p->nVal]; - *ppCursor = &pCur->base; - rc = lsm_csr_open(p->pDb, &pCur->pLsmCur); - if( rc==LSM_OK ){ - rc = SQLITE_OK; - }else{ - sqlite3_free(pCur); - *ppCursor = 0; - rc = SQLITE_ERROR; - } - return rc; -} - -/* -** Destructor for a lsm1_cursor. -*/ -static int lsm1Close(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - sqlite3_free(pCur->pKey2); - lsm_csr_close(pCur->pLsmCur); - sqlite3_free(pCur); - return SQLITE_OK; -} - - -/* -** Advance a lsm1_cursor to its next row of output. -*/ -static int lsm1Next(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - int rc = LSM_OK; - if( pCur->bUnique ){ - pCur->atEof = 1; - }else{ - if( pCur->isDesc ){ - rc = lsm_csr_prev(pCur->pLsmCur); - }else{ - rc = lsm_csr_next(pCur->pLsmCur); - } - if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)==0 ){ - pCur->atEof = 1; - } - if( pCur->pKey2 && pCur->atEof==0 ){ - const u8 *pVal; - u32 nVal; - assert( pCur->isDesc==0 ); - rc = lsm_csr_key(pCur->pLsmCur, (const void**)&pVal, (int*)&nVal); - if( rc==LSM_OK ){ - u32 len = pCur->nKey2; - int c; - if( len>nVal ) len = nVal; - c = memcmp(pVal, pCur->pKey2, len); - if( c==0 ) c = nVal - pCur->nKey2; - if( c>0 ) pCur->atEof = 1; - } - } - pCur->zData = 0; - } - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** Return TRUE if the cursor has been moved off of the last -** row of output. -*/ -static int lsm1Eof(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - return pCur->atEof; -} - -/* -** Rowids are not supported by the underlying virtual table. So always -** return 0 for the rowid. -*/ -static int lsm1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - *pRowid = 0; - return SQLITE_OK; -} - -/* -** Type prefixes on LSM keys -*/ -#define LSM1_TYPE_NEGATIVE 0 -#define LSM1_TYPE_POSITIVE 1 -#define LSM1_TYPE_TEXT 2 -#define LSM1_TYPE_BLOB 3 - -/* -** Write a 32-bit unsigned integer as 4 big-endian bytes. -*/ -static void varintWrite32(unsigned char *z, unsigned int y){ - z[0] = (unsigned char)(y>>24); - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); -} - -/* -** Write a varint into z[]. The buffer z[] must be at least 9 characters -** long to accommodate the largest possible varint. Return the number of -** bytes of z[] used. -*/ -static int lsm1PutVarint64(unsigned char *z, sqlite3_uint64 x){ - unsigned int w, y; - if( x<=240 ){ - z[0] = (unsigned char)x; - return 1; - } - if( x<=2287 ){ - y = (unsigned int)(x - 240); - z[0] = (unsigned char)(y/256 + 241); - z[1] = (unsigned char)(y%256); - return 2; - } - if( x<=67823 ){ - y = (unsigned int)(x - 2288); - z[0] = 249; - z[1] = (unsigned char)(y/256); - z[2] = (unsigned char)(y%256); - return 3; - } - y = (unsigned int)x; - w = (unsigned int)(x>>32); - if( w==0 ){ - if( y<=16777215 ){ - z[0] = 250; - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); - return 4; - } - z[0] = 251; - varintWrite32(z+1, y); - return 5; - } - if( w<=255 ){ - z[0] = 252; - z[1] = (unsigned char)w; - varintWrite32(z+2, y); - return 6; - } - if( w<=65535 ){ - z[0] = 253; - z[1] = (unsigned char)(w>>8); - z[2] = (unsigned char)w; - varintWrite32(z+3, y); - return 7; - } - if( w<=16777215 ){ - z[0] = 254; - z[1] = (unsigned char)(w>>16); - z[2] = (unsigned char)(w>>8); - z[3] = (unsigned char)w; - varintWrite32(z+4, y); - return 8; - } - z[0] = 255; - varintWrite32(z+1, w); - varintWrite32(z+5, y); - return 9; -} - -/* Append non-negative integer x as a variable-length integer. -*/ -static void lsm1VblobAppendVarint(lsm1_vblob *p, sqlite3_uint64 x){ - sqlite3_int64 n = p->n; - if( n+9>p->nAlloc && lsm1VblobEnlarge(p, 9) ) return; - p->n += lsm1PutVarint64(p->a+p->n, x); -} - -/* -** Decode the varint in the first n bytes z[]. Write the integer value -** into *pResult and return the number of bytes in the varint. -** -** If the decode fails because there are not enough bytes in z[] then -** return 0; -*/ -static int lsm1GetVarint64( - const unsigned char *z, - int n, - sqlite3_uint64 *pResult -){ - unsigned int x; - if( n<1 ) return 0; - if( z[0]<=240 ){ - *pResult = z[0]; - return 1; - } - if( z[0]<=248 ){ - if( n<2 ) return 0; - *pResult = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( n<z[0]-246 ) return 0; - if( z[0]==249 ){ - *pResult = 2288 + 256*z[1] + z[2]; - return 3; - } - if( z[0]==250 ){ - *pResult = (z[1]<<16) + (z[2]<<8) + z[3]; - return 4; - } - x = (z[1]<<24) + (z[2]<<16) + (z[3]<<8) + z[4]; - if( z[0]==251 ){ - *pResult = x; - return 5; - } - if( z[0]==252 ){ - *pResult = (((sqlite3_uint64)x)<<8) + z[5]; - return 6; - } - if( z[0]==253 ){ - *pResult = (((sqlite3_uint64)x)<<16) + (z[5]<<8) + z[6]; - return 7; - } - if( z[0]==254 ){ - *pResult = (((sqlite3_uint64)x)<<24) + (z[5]<<16) + (z[6]<<8) + z[7]; - return 8; - } - *pResult = (((sqlite3_uint64)x)<<32) + - (0xffffffff & ((z[5]<<24) + (z[6]<<16) + (z[7]<<8) + z[8])); - return 9; -} - -/* Encoded a signed integer as a varint. Numbers close to zero uses fewer -** bytes than numbers far away from zero. However, the result is not in -** lexicographical order. -** -** Encoding: Non-negative integer X is encoding as an unsigned -** varint X*2. Negative integer Y is encoding as an unsigned -** varint (1-Y)*2 + 1. -*/ -static int lsm1PutSignedVarint64(u8 *z, sqlite3_int64 v){ - sqlite3_uint64 u; - if( v>=0 ){ - u = (sqlite3_uint64)v; - return lsm1PutVarint64(z, u*2); - }else{ - u = (sqlite3_uint64)(-1-v); - return lsm1PutVarint64(z, u*2+1); - } -} - -/* Decoded a signed varint. */ -static int lsm1GetSignedVarint64( - const unsigned char *z, - int n, - sqlite3_int64 *pResult -){ - sqlite3_uint64 u = 0; - n = lsm1GetVarint64(z, n, &u); - if( u&1 ){ - *pResult = -1 - (sqlite3_int64)(u>>1); - }else{ - *pResult = (sqlite3_int64)(u>>1); - } - return n; -} - - -/* -** Read the value part of the key-value pair and decode it into columns. -*/ -static int lsm1DecodeValues(lsm1_cursor *pCur){ - lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); - int i, n; - int rc; - u8 eType; - sqlite3_uint64 v; - - if( pCur->zData ) return 1; - rc = lsm_csr_value(pCur->pLsmCur, (const void**)&pCur->zData, - (int*)&pCur->nData); - if( rc ) return 0; - for(i=n=0; i<pTab->nVal; i++){ - v = 0; - n += lsm1GetVarint64(pCur->zData+n, pCur->nData-n, &v); - pCur->aeType[i] = eType = (u8)(v%6); - if( eType==0 ){ - pCur->aiOfst[i] = (u32)(v/6); - pCur->aiLen[i] = 0; - }else{ - pCur->aiOfst[i] = n; - n += (pCur->aiLen[i] = (u32)(v/6)); - } - if( n>pCur->nData ) break; - } - if( i<pTab->nVal ){ - pCur->zData = 0; - return 0; - } - return 1; -} - -/* -** Return values of columns for the row at which the lsm1_cursor -** is currently pointing. -*/ -static int lsm1Column( - sqlite3_vtab_cursor *cur, /* The cursor */ - sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ -){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - lsm1_vtab *pTab = (lsm1_vtab*)(cur->pVtab); - if( i==0 ){ - /* The key column */ - const void *pVal; - int nVal; - if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - if( pTab->keyType==SQLITE_BLOB ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - }else if( pTab->keyType==SQLITE_TEXT ){ - sqlite3_result_text(ctx,(const char*)pVal, nVal, SQLITE_TRANSIENT); - }else{ - const unsigned char *z = (const unsigned char*)pVal; - sqlite3_uint64 v1; - lsm1GetVarint64(z, nVal, &v1); - sqlite3_result_int64(ctx, (sqlite3_int64)v1); - } - } - }else if( i>pTab->nVal ){ - if( i==pTab->nVal+2 ){ /* lsm1_key */ - const void *pVal; - int nVal; - if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - } - }else if( i==pTab->nVal+3 ){ /* lsm1_value */ - const void *pVal; - int nVal; - if( lsm_csr_value(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - } - } - }else if( lsm1DecodeValues(pCur) ){ - /* The i-th value column (where leftmost is 1) */ - const u8 *zData; - u32 nData; - i--; - zData = pCur->zData + pCur->aiOfst[i]; - nData = pCur->aiLen[i]; - switch( pCur->aeType[i] ){ - case 0: { /* in-line integer */ - sqlite3_result_int(ctx, pCur->aiOfst[i]); - break; - } - case SQLITE_INTEGER: { - sqlite3_int64 v; - lsm1GetSignedVarint64(zData, nData, &v); - sqlite3_result_int64(ctx, v); - break; - } - case SQLITE_FLOAT: { - double v; - if( nData==sizeof(v) ){ - memcpy(&v, zData, sizeof(v)); - sqlite3_result_double(ctx, v); - } - break; - } - case SQLITE_TEXT: { - sqlite3_result_text(ctx, (const char*)zData, nData, SQLITE_TRANSIENT); - break; - } - case SQLITE_BLOB: { - sqlite3_result_blob(ctx, zData, nData, SQLITE_TRANSIENT); - break; - } - default: { - /* A NULL. Do nothing */ - } - } - } - return SQLITE_OK; -} - -/* Parameter "pValue" contains an SQL value that is to be used as -** a key in an LSM table. The type of the key is determined by -** "keyType". Extract the raw bytes used for the key in LSM1. -*/ -static void lsm1KeyFromValue( - int keyType, /* The key type */ - sqlite3_value *pValue, /* The key value */ - u8 *pBuf, /* Storage space for a generated key */ - const u8 **ppKey, /* OUT: the bytes of the key */ - int *pnKey /* OUT: size of the key */ -){ - if( keyType==SQLITE_BLOB ){ - *ppKey = (const u8*)sqlite3_value_blob(pValue); - *pnKey = sqlite3_value_bytes(pValue); - }else if( keyType==SQLITE_TEXT ){ - *ppKey = (const u8*)sqlite3_value_text(pValue); - *pnKey = sqlite3_value_bytes(pValue); - }else{ - sqlite3_int64 v = sqlite3_value_int64(pValue); - if( v<0 ) v = 0; - *pnKey = lsm1PutVarint64(pBuf, v); - *ppKey = pBuf; - } -} - -/* Move to the first row to return. -*/ -static int lsm1Filter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - lsm1_cursor *pCur = (lsm1_cursor *)pVtabCursor; - lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); - int rc = LSM_OK; - int seekType = -1; - const u8 *pVal = 0; - int nVal; - u8 keyType = pTab->keyType; - u8 aKey1[16]; - - pCur->atEof = 1; - sqlite3_free(pCur->pKey2); - pCur->pKey2 = 0; - if( idxNum<99 ){ - lsm1KeyFromValue(keyType, argv[0], aKey1, &pVal, &nVal); - } - switch( idxNum ){ - case 0: { /* key==argv[0] */ - assert( argc==1 ); - seekType = LSM_SEEK_EQ; - pCur->isDesc = 0; - pCur->bUnique = 1; - break; - } - case 1: { /* key>=argv[0] AND key<=argv[1] */ - u8 aKey[12]; - seekType = LSM_SEEK_GE; - pCur->isDesc = 0; - pCur->bUnique = 0; - if( keyType==SQLITE_INTEGER ){ - sqlite3_int64 v = sqlite3_value_int64(argv[1]); - if( v<0 ) v = 0; - pCur->nKey2 = lsm1PutVarint64(aKey, (sqlite3_uint64)v); - pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); - if( pCur->pKey2==0 ) return SQLITE_NOMEM; - memcpy(pCur->pKey2, aKey, pCur->nKey2); - }else{ - pCur->nKey2 = sqlite3_value_bytes(argv[1]); - pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); - if( pCur->pKey2==0 ) return SQLITE_NOMEM; - if( keyType==SQLITE_BLOB ){ - memcpy(pCur->pKey2, sqlite3_value_blob(argv[1]), pCur->nKey2); - }else{ - memcpy(pCur->pKey2, sqlite3_value_text(argv[1]), pCur->nKey2); - } - } - break; - } - case 2: { /* key>=argv[0] */ - seekType = LSM_SEEK_GE; - pCur->isDesc = 0; - pCur->bUnique = 0; - break; - } - case 3: { /* key<=argv[0] */ - seekType = LSM_SEEK_LE; - pCur->isDesc = 1; - pCur->bUnique = 0; - break; - } - default: { /* full table scan */ - pCur->isDesc = 0; - pCur->bUnique = 0; - break; - } - } - if( pVal ){ - rc = lsm_csr_seek(pCur->pLsmCur, pVal, nVal, seekType); - }else{ - rc = lsm_csr_first(pCur->pLsmCur); - } - if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)!=0 ){ - pCur->atEof = 0; - } - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** Only comparisons against the key are allowed. The idxNum defines -** which comparisons are available: -** -** 0 key==?1 -** 1 key>=?1 AND key<=?2 -** 2 key>?1 or key>=?1 -** 3 key<?1 or key<=?1 -** 99 Full table scan only -*/ -static int lsm1BestIndex( - sqlite3_vtab *tab, - sqlite3_index_info *pIdxInfo -){ - int i; /* Loop over constraints */ - int idxNum = 99; /* The query plan */ - int nArg = 0; /* Number of arguments to xFilter */ - int argIdx = -1; /* Index of the key== constraint, or -1 if none */ - int iIdx2 = -1; /* The index of the second key */ - int omit1 = 0; - int omit2 = 0; - - const struct sqlite3_index_constraint *pConstraint; - pConstraint = pIdxInfo->aConstraint; - for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->iColumn!=0 ) continue; - switch( pConstraint->op ){ - case SQLITE_INDEX_CONSTRAINT_EQ: { - if( idxNum>0 ){ - argIdx = i; - iIdx2 = -1; - idxNum = 0; - omit1 = 1; - } - break; - } - case SQLITE_INDEX_CONSTRAINT_GE: - case SQLITE_INDEX_CONSTRAINT_GT: { - if( idxNum==99 ){ - argIdx = i; - idxNum = 2; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; - }else if( idxNum==3 ){ - iIdx2 = idxNum; - omit2 = omit1; - argIdx = i; - idxNum = 1; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; - } - break; - } - case SQLITE_INDEX_CONSTRAINT_LE: - case SQLITE_INDEX_CONSTRAINT_LT: { - if( idxNum==99 ){ - argIdx = i; - idxNum = 3; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; - }else if( idxNum==2 ){ - iIdx2 = i; - idxNum = 1; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; - } - break; - } - } - } - if( argIdx>=0 ){ - pIdxInfo->aConstraintUsage[argIdx].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[argIdx].omit = omit1; - } - if( iIdx2>=0 ){ - pIdxInfo->aConstraintUsage[iIdx2].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[iIdx2].omit = omit2; - } - if( idxNum==0 ){ - pIdxInfo->estimatedCost = (double)1; - pIdxInfo->estimatedRows = 1; - pIdxInfo->orderByConsumed = 1; - }else if( idxNum==1 ){ - pIdxInfo->estimatedCost = (double)100; - pIdxInfo->estimatedRows = 100; - }else if( idxNum<99 ){ - pIdxInfo->estimatedCost = (double)5000; - pIdxInfo->estimatedRows = 5000; - }else{ - /* Full table scan */ - pIdxInfo->estimatedCost = (double)2147483647; - pIdxInfo->estimatedRows = 2147483647; - } - pIdxInfo->idxNum = idxNum; - return SQLITE_OK; -} - -/* -** The xUpdate method is normally used for INSERT, REPLACE, UPDATE, and -** DELETE. But this virtual table only supports INSERT and REPLACE. -** DELETE is accomplished by inserting a record with a value of NULL. -** UPDATE is achieved by using REPLACE. -*/ -int lsm1Update( - sqlite3_vtab *pVTab, - int argc, - sqlite3_value **argv, - sqlite_int64 *pRowid -){ - lsm1_vtab *p = (lsm1_vtab*)pVTab; - int nKey, nKey2; - int i; - int rc = LSM_OK; - const u8 *pKey, *pKey2; - unsigned char aKey[16]; - unsigned char pSpace[16]; - lsm1_vblob val; - - if( argc==1 ){ - /* DELETE the record whose key is argv[0] */ - lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); - lsm_delete(p->pDb, pKey, nKey); - return SQLITE_OK; - } - - if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ - /* An UPDATE */ - lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); - lsm1KeyFromValue(p->keyType, argv[1], pSpace, &pKey2, &nKey2); - if( nKey!=nKey2 || memcmp(pKey, pKey2, nKey)!=0 ){ - /* The UPDATE changes the PRIMARY KEY value. DELETE the old key */ - lsm_delete(p->pDb, pKey, nKey); - } - /* Fall through into the INSERT case to complete the UPDATE */ - } - - /* "INSERT INTO tab(lsm1_command) VALUES('....')" is used to implement - ** special commands. - */ - if( sqlite3_value_type(argv[3+p->nVal])!=SQLITE_NULL ){ - return SQLITE_OK; - } - lsm1KeyFromValue(p->keyType, argv[2], aKey, &pKey, &nKey); - memset(&val, 0, sizeof(val)); - for(i=0; i<p->nVal; i++){ - sqlite3_value *pArg = argv[3+i]; - u8 eType = sqlite3_value_type(pArg); - switch( eType ){ - case SQLITE_NULL: { - lsm1VblobAppendVarint(&val, SQLITE_NULL); - break; - } - case SQLITE_INTEGER: { - sqlite3_int64 v = sqlite3_value_int64(pArg); - if( v>=0 && v<=240/6 ){ - lsm1VblobAppendVarint(&val, v*6); - }else{ - int n = lsm1PutSignedVarint64(pSpace, v); - lsm1VblobAppendVarint(&val, SQLITE_INTEGER + n*6); - lsm1VblobAppend(&val, pSpace, n); - } - break; - } - case SQLITE_FLOAT: { - double r = sqlite3_value_double(pArg); - lsm1VblobAppendVarint(&val, SQLITE_FLOAT + 8*6); - lsm1VblobAppend(&val, (u8*)&r, sizeof(r)); - break; - } - case SQLITE_BLOB: { - int n = sqlite3_value_bytes(pArg); - lsm1VblobAppendVarint(&val, n*6 + SQLITE_BLOB); - lsm1VblobAppend(&val, sqlite3_value_blob(pArg), n); - break; - } - case SQLITE_TEXT: { - int n = sqlite3_value_bytes(pArg); - lsm1VblobAppendVarint(&val, n*6 + SQLITE_TEXT); - lsm1VblobAppend(&val, sqlite3_value_text(pArg), n); - break; - } - } - } - if( val.errNoMem ){ - return SQLITE_NOMEM; - } - rc = lsm_insert(p->pDb, pKey, nKey, val.a, val.n); - sqlite3_free(val.a); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Begin a transaction -*/ -static int lsm1Begin(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_begin(p->pDb, 1); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Phase 1 of a transaction commit. -*/ -static int lsm1Sync(sqlite3_vtab *pVtab){ - return SQLITE_OK; -} - -/* Commit a transaction -*/ -static int lsm1Commit(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_commit(p->pDb, 0); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Rollback a transaction -*/ -static int lsm1Rollback(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_rollback(p->pDb, 0); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** This following structure defines all the methods for the -** generate_lsm1 virtual table. -*/ -static sqlite3_module lsm1Module = { - 0, /* iVersion */ - lsm1Connect, /* xCreate */ - lsm1Connect, /* xConnect */ - lsm1BestIndex, /* xBestIndex */ - lsm1Disconnect, /* xDisconnect */ - lsm1Disconnect, /* xDestroy */ - lsm1Open, /* xOpen - open a cursor */ - lsm1Close, /* xClose - close a cursor */ - lsm1Filter, /* xFilter - configure scan constraints */ - lsm1Next, /* xNext - advance a cursor */ - lsm1Eof, /* xEof - check for end of scan */ - lsm1Column, /* xColumn - read data */ - lsm1Rowid, /* xRowid - read data */ - lsm1Update, /* xUpdate */ - lsm1Begin, /* xBegin */ - lsm1Sync, /* xSync */ - lsm1Commit, /* xCommit */ - lsm1Rollback, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - 0 /* xIntegrity */ -}; - - -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_lsm_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - rc = sqlite3_create_module(db, "lsm1", &lsm1Module, 0); - return rc; -} diff --git a/ext/lsm1/lsm_win32.c b/ext/lsm1/lsm_win32.c deleted file mode 100644 index 6c5d06b4c..000000000 --- a/ext/lsm1/lsm_win32.c +++ /dev/null @@ -1,1063 +0,0 @@ -/* -** 2011-12-03 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Win32-specific run-time environment implementation for LSM. -*/ - -#ifdef _WIN32 - -#include <assert.h> -#include <string.h> - -#include <stdlib.h> -#include <stdarg.h> -#include <stdio.h> -#include <ctype.h> - -#include "windows.h" - -#include "lsmInt.h" - -/* -** An open file is an instance of the following object -*/ -typedef struct Win32File Win32File; -struct Win32File { - lsm_env *pEnv; /* The run-time environment */ - const char *zName; /* Full path to file */ - - HANDLE hFile; /* Open file handle */ - HANDLE hShmFile; /* File handle for *-shm file */ - - SYSTEM_INFO sysInfo; /* Operating system information */ - HANDLE hMap; /* File handle for mapping */ - LPVOID pMap; /* Pointer to mapping of file fd */ - size_t nMap; /* Size of mapping at pMap in bytes */ - int nShm; /* Number of entries in ahShm[]/apShm[] */ - LPHANDLE ahShm; /* Array of handles for shared mappings */ - LPVOID *apShm; /* Array of 32K shared memory segments */ -}; - -static char *win32ShmFile(Win32File *pWin32File){ - char *zShm; - int nName = strlen(pWin32File->zName); - zShm = (char *)lsmMallocZero(pWin32File->pEnv, nName+4+1); - if( zShm ){ - memcpy(zShm, pWin32File->zName, nName); - memcpy(&zShm[nName], "-shm", 5); - } - return zShm; -} - -static int win32Sleep(int us){ - Sleep((us + 999) / 1000); - return LSM_OK; -} - -/* -** The number of times that an I/O operation will be retried following a -** locking error - probably caused by antivirus software. Also the initial -** delay before the first retry. The delay increases linearly with each -** retry. -*/ -#ifndef LSM_WIN32_IOERR_RETRY -# define LSM_WIN32_IOERR_RETRY 10 -#endif -#ifndef LSM_WIN32_IOERR_RETRY_DELAY -# define LSM_WIN32_IOERR_RETRY_DELAY 25000 -#endif -static int win32IoerrRetry = LSM_WIN32_IOERR_RETRY; -static int win32IoerrRetryDelay = LSM_WIN32_IOERR_RETRY_DELAY; - -/* -** The "win32IoerrCanRetry1" macro is used to determine if a particular -** I/O error code obtained via GetLastError() is eligible to be retried. -** It must accept the error code DWORD as its only argument and should -** return non-zero if the error code is transient in nature and the -** operation responsible for generating the original error might succeed -** upon being retried. The argument to this macro should be a variable. -** -** Additionally, a macro named "win32IoerrCanRetry2" may be defined. If -** it is defined, it will be consulted only when the macro -** "win32IoerrCanRetry1" returns zero. The "win32IoerrCanRetry2" macro -** is completely optional and may be used to include additional error -** codes in the set that should result in the failing I/O operation being -** retried by the caller. If defined, the "win32IoerrCanRetry2" macro -** must exhibit external semantics identical to those of the -** "win32IoerrCanRetry1" macro. -*/ -#if !defined(win32IoerrCanRetry1) -#define win32IoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ - ((a)==ERROR_SHARING_VIOLATION) || \ - ((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_DEV_NOT_EXIST) || \ - ((a)==ERROR_NETNAME_DELETED) || \ - ((a)==ERROR_SEM_TIMEOUT) || \ - ((a)==ERROR_NETWORK_UNREACHABLE)) -#endif - -/* -** If an I/O error occurs, invoke this routine to see if it should be -** retried. Return TRUE to retry. Return FALSE to give up with an -** error. -*/ -static int win32RetryIoerr( - lsm_env *pEnv, - int *pnRetry -){ - DWORD lastErrno; - if( *pnRetry>=win32IoerrRetry ){ - return 0; - } - lastErrno = GetLastError(); - if( win32IoerrCanRetry1(lastErrno) ){ - win32Sleep(win32IoerrRetryDelay*(1+*pnRetry)); - ++*pnRetry; - return 1; - } -#if defined(win32IoerrCanRetry2) - else if( win32IoerrCanRetry2(lastErrno) ){ - win32Sleep(win32IoerrRetryDelay*(1+*pnRetry)); - ++*pnRetry; - return 1; - } -#endif - return 0; -} - -/* -** Convert a UTF-8 string to Microsoft Unicode. -** -** Space to hold the returned string is obtained from lsmMalloc(). -*/ -static LPWSTR win32Utf8ToUnicode(lsm_env *pEnv, const char *zText){ - int nChar; - LPWSTR zWideText; - - nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); - if( nChar==0 ){ - return 0; - } - zWideText = lsmMallocZero(pEnv, nChar * sizeof(WCHAR)); - if( zWideText==0 ){ - return 0; - } - nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, nChar); - if( nChar==0 ){ - lsmFree(pEnv, zWideText); - zWideText = 0; - } - return zWideText; -} - -/* -** Convert a Microsoft Unicode string to UTF-8. -** -** Space to hold the returned string is obtained from lsmMalloc(). -*/ -static char *win32UnicodeToUtf8(lsm_env *pEnv, LPCWSTR zWideText){ - int nByte; - char *zText; - - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); - if( nByte == 0 ){ - return 0; - } - zText = lsmMallocZero(pEnv, nByte); - if( zText==0 ){ - return 0; - } - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, 0, 0); - if( nByte == 0 ){ - lsmFree(pEnv, zText); - zText = 0; - } - return zText; -} - -#if !defined(win32IsNotFound) -#define win32IsNotFound(a) (((a)==ERROR_FILE_NOT_FOUND) || \ - ((a)==ERROR_PATH_NOT_FOUND)) -#endif - -static int win32Open( - lsm_env *pEnv, - const char *zFile, - int flags, - LPHANDLE phFile -){ - int rc; - LPWSTR zConverted; - - zConverted = win32Utf8ToUnicode(pEnv, zFile); - if( zConverted==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - int bReadonly = (flags & LSM_OPEN_READONLY); - DWORD dwDesiredAccess; - DWORD dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; - DWORD dwCreationDisposition; - DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; - HANDLE hFile; - int nRetry = 0; - if( bReadonly ){ - dwDesiredAccess = GENERIC_READ; - dwCreationDisposition = OPEN_EXISTING; - }else{ - dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; - dwCreationDisposition = OPEN_ALWAYS; - } - while( (hFile = CreateFileW((LPCWSTR)zConverted, - dwDesiredAccess, - dwShareMode, NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL))==INVALID_HANDLE_VALUE && - win32RetryIoerr(pEnv, &nRetry) ){ - /* Noop */ - } - lsmFree(pEnv, zConverted); - if( hFile!=INVALID_HANDLE_VALUE ){ - *phFile = hFile; - rc = LSM_OK; - }else{ - if( win32IsNotFound(GetLastError()) ){ - rc = lsmErrorBkpt(LSM_IOERR_NOENT); - }else{ - rc = LSM_IOERR_BKPT; - } - } - } - return rc; -} - -static int lsmWin32OsOpen( - lsm_env *pEnv, - const char *zFile, - int flags, - lsm_file **ppFile -){ - int rc = LSM_OK; - Win32File *pWin32File; - - pWin32File = lsmMallocZero(pEnv, sizeof(Win32File)); - if( pWin32File==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - HANDLE hFile = NULL; - - rc = win32Open(pEnv, zFile, flags, &hFile); - if( rc==LSM_OK ){ - memset(&pWin32File->sysInfo, 0, sizeof(SYSTEM_INFO)); - GetSystemInfo(&pWin32File->sysInfo); - pWin32File->pEnv = pEnv; - pWin32File->zName = zFile; - pWin32File->hFile = hFile; - }else{ - lsmFree(pEnv, pWin32File); - pWin32File = 0; - } - } - *ppFile = (lsm_file *)pWin32File; - return rc; -} - -static int lsmWin32OsWrite( - lsm_file *pFile, /* File to write to */ - lsm_i64 iOff, /* Offset to write to */ - void *pData, /* Write data from this buffer */ - int nData /* Bytes of data to write */ -){ - Win32File *pWin32File = (Win32File *)pFile; - OVERLAPPED overlapped; /* The offset for WriteFile. */ - u8 *aRem = (u8 *)pData; /* Data yet to be written */ - int nRem = nData; /* Number of bytes yet to be written */ - int nRetry = 0; /* Number of retrys */ - - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = (LONG)(iOff & 0XFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0x7FFFFFFF); - while( nRem>0 ){ - DWORD nWrite = 0; /* Bytes written using WriteFile */ - if( !WriteFile(pWin32File->hFile, aRem, nRem, &nWrite, &overlapped) ){ - if( win32RetryIoerr(pWin32File->pEnv, &nRetry) ) continue; - break; - } - assert( nWrite==0 || nWrite<=(DWORD)nRem ); - if( nWrite==0 || nWrite>(DWORD)nRem ){ - break; - } - iOff += nWrite; - overlapped.Offset = (LONG)(iOff & 0xFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0x7FFFFFFF); - aRem += nWrite; - nRem -= nWrite; - } - if( nRem!=0 ) return LSM_IOERR_BKPT; - return LSM_OK; -} - -static int win32Truncate( - HANDLE hFile, - lsm_i64 nSize -){ - LARGE_INTEGER offset; - offset.QuadPart = nSize; - if( !SetFilePointerEx(hFile, offset, 0, FILE_BEGIN) ){ - return LSM_IOERR_BKPT; - } - if (!SetEndOfFile(hFile) ){ - return LSM_IOERR_BKPT; - } - return LSM_OK; -} - -static int lsmWin32OsTruncate( - lsm_file *pFile, /* File to write to */ - lsm_i64 nSize /* Size to truncate file to */ -){ - Win32File *pWin32File = (Win32File *)pFile; - return win32Truncate(pWin32File->hFile, nSize); -} - -static int lsmWin32OsRead( - lsm_file *pFile, /* File to read from */ - lsm_i64 iOff, /* Offset to read from */ - void *pData, /* Read data into this buffer */ - int nData /* Bytes of data to read */ -){ - Win32File *pWin32File = (Win32File *)pFile; - OVERLAPPED overlapped; /* The offset for ReadFile */ - DWORD nRead = 0; /* Bytes read using ReadFile */ - int nRetry = 0; /* Number of retrys */ - - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = (LONG)(iOff & 0XFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0X7FFFFFFF); - while( !ReadFile(pWin32File->hFile, pData, nData, &nRead, &overlapped) && - GetLastError()!=ERROR_HANDLE_EOF ){ - if( win32RetryIoerr(pWin32File->pEnv, &nRetry) ) continue; - return LSM_IOERR_BKPT; - } - if( nRead<(DWORD)nData ){ - /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pData)[nRead], 0, nData - nRead); - } - return LSM_OK; -} - -static int lsmWin32OsSync(lsm_file *pFile){ - int rc = LSM_OK; - -#ifndef LSM_NO_SYNC - Win32File *pWin32File = (Win32File *)pFile; - - if( pWin32File->pMap!=NULL ){ - if( !FlushViewOfFile(pWin32File->pMap, 0) ){ - rc = LSM_IOERR_BKPT; - } - } - if( rc==LSM_OK && !FlushFileBuffers(pWin32File->hFile) ){ - rc = LSM_IOERR_BKPT; - } -#else - unused_parameter(pFile); -#endif - - return rc; -} - -static int lsmWin32OsSectorSize(lsm_file *pFile){ - return 512; -} - -static void win32Unmap(Win32File *pWin32File){ - if( pWin32File->pMap!=NULL ){ - UnmapViewOfFile(pWin32File->pMap); - pWin32File->pMap = NULL; - pWin32File->nMap = 0; - } - if( pWin32File->hMap!=NULL ){ - CloseHandle(pWin32File->hMap); - pWin32File->hMap = NULL; - } -} - -static int lsmWin32OsRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - Win32File *pWin32File = (Win32File *)pFile; - - /* If the file is between 0 and 2MB in size, extend it in chunks of 256K. - ** Thereafter, in chunks of 1MB at a time. */ - const int aIncrSz[] = {256*1024, 1024*1024}; - int nIncrSz = aIncrSz[iMin>(2*1024*1024)]; - - *ppOut = NULL; - *pnOut = 0; - - win32Unmap(pWin32File); - if( iMin>=0 ){ - LARGE_INTEGER fileSize; - DWORD dwSizeHigh; - DWORD dwSizeLow; - HANDLE hMap; - LPVOID pMap; - memset(&fileSize, 0, sizeof(LARGE_INTEGER)); - if( !GetFileSizeEx(pWin32File->hFile, &fileSize) ){ - return LSM_IOERR_BKPT; - } - assert( fileSize.QuadPart>=0 ); - if( fileSize.QuadPart<iMin ){ - int rc; - fileSize.QuadPart = ((iMin + nIncrSz-1) / nIncrSz) * nIncrSz; - rc = lsmWin32OsTruncate(pFile, fileSize.QuadPart); - if( rc!=LSM_OK ){ - return rc; - } - } - dwSizeLow = (DWORD)(fileSize.QuadPart & 0xFFFFFFFF); - dwSizeHigh = (DWORD)((fileSize.QuadPart & 0x7FFFFFFFFFFFFFFF) >> 32); - hMap = CreateFileMappingW(pWin32File->hFile, NULL, PAGE_READWRITE, - dwSizeHigh, dwSizeLow, NULL); - if( hMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->hMap = hMap; - assert( fileSize.QuadPart<=0xFFFFFFFF ); - pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, 0, 0, - (SIZE_T)fileSize.QuadPart); - if( pMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->pMap = pMap; - pWin32File->nMap = (SIZE_T)fileSize.QuadPart; - } - *ppOut = pWin32File->pMap; - *pnOut = pWin32File->nMap; - return LSM_OK; -} - -static BOOL win32IsDriveLetterAndColon( - const char *zPathname -){ - return ( isalpha(zPathname[0]) && zPathname[1]==':' ); -} - -static int lsmWin32OsFullpath( - lsm_env *pEnv, - const char *zName, - char *zOut, - int *pnOut -){ - DWORD nByte; - void *zConverted; - LPWSTR zTempWide; - char *zTempUtf8; - - if( zName[0]=='/' && win32IsDriveLetterAndColon(zName+1) ){ - zName++; - } - zConverted = win32Utf8ToUnicode(pEnv, zName); - if( zConverted==0 ){ - return LSM_NOMEM_BKPT; - } - nByte = GetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); - if( nByte==0 ){ - lsmFree(pEnv, zConverted); - return LSM_IOERR_BKPT; - } - nByte += 3; - zTempWide = lsmMallocZero(pEnv, nByte * sizeof(zTempWide[0])); - if( zTempWide==0 ){ - lsmFree(pEnv, zConverted); - return LSM_NOMEM_BKPT; - } - nByte = GetFullPathNameW((LPCWSTR)zConverted, nByte, zTempWide, 0); - if( nByte==0 ){ - lsmFree(pEnv, zConverted); - lsmFree(pEnv, zTempWide); - return LSM_IOERR_BKPT; - } - lsmFree(pEnv, zConverted); - zTempUtf8 = win32UnicodeToUtf8(pEnv, zTempWide); - lsmFree(pEnv, zTempWide); - if( zTempUtf8 ){ - int nOut = *pnOut; - int nLen = strlen(zTempUtf8) + 1; - if( nLen<=nOut ){ - snprintf(zOut, nOut, "%s", zTempUtf8); - } - lsmFree(pEnv, zTempUtf8); - *pnOut = nLen; - return LSM_OK; - }else{ - return LSM_NOMEM_BKPT; - } -} - -static int lsmWin32OsFileid( - lsm_file *pFile, - void *pBuf, - int *pnBuf -){ - int nBuf; - int nReq; - u8 *pBuf2 = (u8 *)pBuf; - Win32File *pWin32File = (Win32File *)pFile; - BY_HANDLE_FILE_INFORMATION fileInfo; - - nBuf = *pnBuf; - nReq = (sizeof(fileInfo.dwVolumeSerialNumber) + - sizeof(fileInfo.nFileIndexHigh) + - sizeof(fileInfo.nFileIndexLow)); - *pnBuf = nReq; - if( nReq>nBuf ) return LSM_OK; - memset(&fileInfo, 0, sizeof(BY_HANDLE_FILE_INFORMATION)); - if( !GetFileInformationByHandle(pWin32File->hFile, &fileInfo) ){ - return LSM_IOERR_BKPT; - } - nReq = sizeof(fileInfo.dwVolumeSerialNumber); - memcpy(pBuf2, &fileInfo.dwVolumeSerialNumber, nReq); - pBuf2 += nReq; - nReq = sizeof(fileInfo.nFileIndexHigh); - memcpy(pBuf, &fileInfo.nFileIndexHigh, nReq); - pBuf2 += nReq; - nReq = sizeof(fileInfo.nFileIndexLow); - memcpy(pBuf2, &fileInfo.nFileIndexLow, nReq); - return LSM_OK; -} - -static int win32Delete( - lsm_env *pEnv, - const char *zFile -){ - int rc; - LPWSTR zConverted; - - zConverted = win32Utf8ToUnicode(pEnv, zFile); - if( zConverted==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - int nRetry = 0; - DWORD attr; - - do { - attr = GetFileAttributesW(zConverted); - if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = LSM_IOERR_BKPT; - break; - } - if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ - rc = LSM_IOERR_BKPT; /* Files only. */ - break; - } - if ( DeleteFileW(zConverted) ){ - rc = LSM_OK; /* Deleted OK. */ - break; - } - if ( !win32RetryIoerr(pEnv, &nRetry) ){ - rc = LSM_IOERR_BKPT; /* No more retries. */ - break; - } - }while( 1 ); - } - lsmFree(pEnv, zConverted); - return rc; -} - -static int lsmWin32OsUnlink(lsm_env *pEnv, const char *zFile){ - return win32Delete(pEnv, zFile); -} - -#if !defined(win32IsLockBusy) -#define win32IsLockBusy(a) (((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_IO_PENDING)) -#endif - -static int win32LockFile( - Win32File *pWin32File, - int iLock, - int nLock, - int eType -){ - OVERLAPPED ovlp; - - assert( LSM_LOCK_UNLOCK==0 ); - assert( LSM_LOCK_SHARED==1 ); - assert( LSM_LOCK_EXCL==2 ); - assert( eType>=LSM_LOCK_UNLOCK && eType<=LSM_LOCK_EXCL ); - assert( nLock>=0 ); - assert( iLock>0 && iLock<=32 ); - - memset(&ovlp, 0, sizeof(OVERLAPPED)); - ovlp.Offset = (4096-iLock-nLock+1); - if( eType>LSM_LOCK_UNLOCK ){ - DWORD flags = LOCKFILE_FAIL_IMMEDIATELY; - if( eType>=LSM_LOCK_EXCL ) flags |= LOCKFILE_EXCLUSIVE_LOCK; - if( !LockFileEx(pWin32File->hFile, flags, 0, (DWORD)nLock, 0, &ovlp) ){ - if( win32IsLockBusy(GetLastError()) ){ - return LSM_BUSY; - }else{ - return LSM_IOERR_BKPT; - } - } - }else{ - if( !UnlockFileEx(pWin32File->hFile, 0, (DWORD)nLock, 0, &ovlp) ){ - return LSM_IOERR_BKPT; - } - } - return LSM_OK; -} - -static int lsmWin32OsLock(lsm_file *pFile, int iLock, int eType){ - Win32File *pWin32File = (Win32File *)pFile; - return win32LockFile(pWin32File, iLock, 1, eType); -} - -static int lsmWin32OsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - int rc; - Win32File *pWin32File = (Win32File *)pFile; - rc = win32LockFile(pWin32File, iLock, nLock, eType); - if( rc!=LSM_OK ) return rc; - win32LockFile(pWin32File, iLock, nLock, LSM_LOCK_UNLOCK); - return LSM_OK; -} - -static int lsmWin32OsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ - int rc; - Win32File *pWin32File = (Win32File *)pFile; - int iOffset = iChunk * sz; - int iOffsetShift = iOffset % pWin32File->sysInfo.dwAllocationGranularity; - int nNew = iChunk + 1; - lsm_i64 nReq = nNew * sz; - - *ppShm = NULL; - assert( sz>=0 ); - assert( sz==LSM_SHM_CHUNK_SIZE ); - if( iChunk>=pWin32File->nShm ){ - LPHANDLE ahNew; - LPVOID *apNew; - LARGE_INTEGER fileSize; - - /* If the shared-memory file has not been opened, open it now. */ - if( pWin32File->hShmFile==NULL ){ - char *zShm = win32ShmFile(pWin32File); - if( !zShm ) return LSM_NOMEM_BKPT; - rc = win32Open(pWin32File->pEnv, zShm, 0, &pWin32File->hShmFile); - lsmFree(pWin32File->pEnv, zShm); - if( rc!=LSM_OK ){ - return rc; - } - } - - /* If the shared-memory file is not large enough to contain the - ** requested chunk, cause it to grow. */ - memset(&fileSize, 0, sizeof(LARGE_INTEGER)); - if( !GetFileSizeEx(pWin32File->hShmFile, &fileSize) ){ - return LSM_IOERR_BKPT; - } - assert( fileSize.QuadPart>=0 ); - if( fileSize.QuadPart<nReq ){ - rc = win32Truncate(pWin32File->hShmFile, nReq); - if( rc!=LSM_OK ){ - return rc; - } - } - - ahNew = (LPHANDLE)lsmMallocZero(pWin32File->pEnv, sizeof(HANDLE) * nNew); - if( !ahNew ) return LSM_NOMEM_BKPT; - apNew = (LPVOID *)lsmMallocZero(pWin32File->pEnv, sizeof(LPVOID) * nNew); - if( !apNew ){ - lsmFree(pWin32File->pEnv, ahNew); - return LSM_NOMEM_BKPT; - } - memcpy(ahNew, pWin32File->ahShm, sizeof(HANDLE) * pWin32File->nShm); - memcpy(apNew, pWin32File->apShm, sizeof(LPVOID) * pWin32File->nShm); - lsmFree(pWin32File->pEnv, pWin32File->ahShm); - pWin32File->ahShm = ahNew; - lsmFree(pWin32File->pEnv, pWin32File->apShm); - pWin32File->apShm = apNew; - pWin32File->nShm = nNew; - } - - if( pWin32File->ahShm[iChunk]==NULL ){ - HANDLE hMap; - assert( nReq<=0xFFFFFFFF ); - hMap = CreateFileMappingW(pWin32File->hShmFile, NULL, PAGE_READWRITE, 0, - (DWORD)nReq, NULL); - if( hMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->ahShm[iChunk] = hMap; - } - if( pWin32File->apShm[iChunk]==NULL ){ - LPVOID pMap; - pMap = MapViewOfFile(pWin32File->ahShm[iChunk], - FILE_MAP_WRITE | FILE_MAP_READ, 0, - iOffset - iOffsetShift, sz + iOffsetShift); - if( pMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->apShm[iChunk] = pMap; - } - if( iOffsetShift!=0 ){ - char *p = (char *)pWin32File->apShm[iChunk]; - *ppShm = (void *)&p[iOffsetShift]; - }else{ - *ppShm = pWin32File->apShm[iChunk]; - } - return LSM_OK; -} - -static void lsmWin32OsShmBarrier(void){ - MemoryBarrier(); -} - -static int lsmWin32OsShmUnmap(lsm_file *pFile, int bDelete){ - Win32File *pWin32File = (Win32File *)pFile; - - if( pWin32File->hShmFile!=NULL ){ - int i; - for(i=0; i<pWin32File->nShm; i++){ - if( pWin32File->apShm[i]!=NULL ){ - UnmapViewOfFile(pWin32File->apShm[i]); - pWin32File->apShm[i] = NULL; - } - if( pWin32File->ahShm[i]!=NULL ){ - CloseHandle(pWin32File->ahShm[i]); - pWin32File->ahShm[i] = NULL; - } - } - CloseHandle(pWin32File->hShmFile); - pWin32File->hShmFile = NULL; - if( bDelete ){ - char *zShm = win32ShmFile(pWin32File); - if( zShm ){ win32Delete(pWin32File->pEnv, zShm); } - lsmFree(pWin32File->pEnv, zShm); - } - } - return LSM_OK; -} - -#define MX_CLOSE_ATTEMPT 3 -static int lsmWin32OsClose(lsm_file *pFile){ - int rc; - int nRetry = 0; - Win32File *pWin32File = (Win32File *)pFile; - lsmWin32OsShmUnmap(pFile, 0); - win32Unmap(pWin32File); - do{ - if( pWin32File->hFile==NULL ){ - rc = LSM_IOERR_BKPT; - break; - } - rc = CloseHandle(pWin32File->hFile); - if( rc ){ - pWin32File->hFile = NULL; - rc = LSM_OK; - break; - } - if( ++nRetry>=MX_CLOSE_ATTEMPT ){ - rc = LSM_IOERR_BKPT; - break; - } - }while( 1 ); - lsmFree(pWin32File->pEnv, pWin32File->ahShm); - lsmFree(pWin32File->pEnv, pWin32File->apShm); - lsmFree(pWin32File->pEnv, pWin32File); - return rc; -} - -static int lsmWin32OsSleep(lsm_env *pEnv, int us){ - unused_parameter(pEnv); - return win32Sleep(us); -} - -/**************************************************************************** -** Memory allocation routines. -*/ - -static void *lsmWin32OsMalloc(lsm_env *pEnv, size_t N){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - return HeapAlloc(GetProcessHeap(), 0, (SIZE_T)N); -} - -static void lsmWin32OsFree(lsm_env *pEnv, void *p){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - if( p ){ - HeapFree(GetProcessHeap(), 0, p); - } -} - -static void *lsmWin32OsRealloc(lsm_env *pEnv, void *p, size_t N){ - unsigned char *m = (unsigned char *)p; - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - if( 1>N ){ - lsmWin32OsFree(pEnv, p); - return NULL; - }else if( NULL==p ){ - return lsmWin32OsMalloc(pEnv, N); - }else{ -#if 0 /* arguable: don't shrink */ - SIZE_T sz = HeapSize(GetProcessHeap(), 0, m); - if( sz>=(SIZE_T)N ){ - return p; - } -#endif - return HeapReAlloc(GetProcessHeap(), 0, m, N); - } -} - -static size_t lsmWin32OsMSize(lsm_env *pEnv, void *p){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - return (size_t)HeapSize(GetProcessHeap(), 0, p); -} - - -#ifdef LSM_MUTEX_WIN32 -/************************************************************************* -** Mutex methods for Win32 based systems. If LSM_MUTEX_WIN32 is -** missing then a no-op implementation of mutexes found below will be -** used instead. -*/ -#include "windows.h" - -typedef struct Win32Mutex Win32Mutex; -struct Win32Mutex { - lsm_env *pEnv; - CRITICAL_SECTION mutex; -#ifdef LSM_DEBUG - DWORD owner; -#endif -}; - -#ifndef WIN32_MUTEX_INITIALIZER -# define WIN32_MUTEX_INITIALIZER { 0 } -#endif - -#ifdef LSM_DEBUG -# define LSM_WIN32_STATIC_MUTEX { 0, WIN32_MUTEX_INITIALIZER, 0 } -#else -# define LSM_WIN32_STATIC_MUTEX { 0, WIN32_MUTEX_INITIALIZER } -#endif - -static int lsmWin32OsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - static volatile LONG initialized = 0; - static Win32Mutex sMutex[2] = { - LSM_WIN32_STATIC_MUTEX, - LSM_WIN32_STATIC_MUTEX - }; - - assert( iMutex==LSM_MUTEX_GLOBAL || iMutex==LSM_MUTEX_HEAP ); - assert( LSM_MUTEX_GLOBAL==1 && LSM_MUTEX_HEAP==2 ); - - if( InterlockedCompareExchange(&initialized, 1, 0)==0 ){ - int i; - for(i=0; i<array_size(sMutex); i++){ - InitializeCriticalSection(&sMutex[i].mutex); - } - } - *ppStatic = (lsm_mutex *)&sMutex[iMutex-1]; - return LSM_OK; -} - -static int lsmWin32OsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - Win32Mutex *pMutex; /* Pointer to new mutex */ - - pMutex = (Win32Mutex *)lsmMallocZero(pEnv, sizeof(Win32Mutex)); - if( !pMutex ) return LSM_NOMEM_BKPT; - - pMutex->pEnv = pEnv; - InitializeCriticalSection(&pMutex->mutex); - - *ppNew = (lsm_mutex *)pMutex; - return LSM_OK; -} - -static void lsmWin32OsMutexDel(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - DeleteCriticalSection(&pMutex->mutex); - lsmFree(pMutex->pEnv, pMutex); -} - -static void lsmWin32OsMutexEnter(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - EnterCriticalSection(&pMutex->mutex); - -#ifdef LSM_DEBUG - assert( pMutex->owner!=GetCurrentThreadId() ); - pMutex->owner = GetCurrentThreadId(); - assert( pMutex->owner==GetCurrentThreadId() ); -#endif -} - -static int lsmWin32OsMutexTry(lsm_mutex *p){ - BOOL bRet; - Win32Mutex *pMutex = (Win32Mutex *)p; - bRet = TryEnterCriticalSection(&pMutex->mutex); -#ifdef LSM_DEBUG - if( bRet ){ - assert( pMutex->owner!=GetCurrentThreadId() ); - pMutex->owner = GetCurrentThreadId(); - assert( pMutex->owner==GetCurrentThreadId() ); - } -#endif - return !bRet; -} - -static void lsmWin32OsMutexLeave(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; -#ifdef LSM_DEBUG - assert( pMutex->owner==GetCurrentThreadId() ); - pMutex->owner = 0; - assert( pMutex->owner!=GetCurrentThreadId() ); -#endif - LeaveCriticalSection(&pMutex->mutex); -} - -#ifdef LSM_DEBUG -static int lsmWin32OsMutexHeld(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - return pMutex ? pMutex->owner==GetCurrentThreadId() : 1; -} -static int lsmWin32OsMutexNotHeld(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - return pMutex ? pMutex->owner!=GetCurrentThreadId() : 1; -} -#endif -/* -** End of Win32 mutex implementation. -*************************************************************************/ -#else -/************************************************************************* -** Noop mutex implementation -*/ -typedef struct NoopMutex NoopMutex; -struct NoopMutex { - lsm_env *pEnv; /* Environment handle (for xFree()) */ - int bHeld; /* True if mutex is held */ - int bStatic; /* True for a static mutex */ -}; -static NoopMutex aStaticNoopMutex[2] = { - {0, 0, 1}, - {0, 0, 1}, -}; - -static int lsmWin32OsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - assert( iMutex>=1 && iMutex<=(int)array_size(aStaticNoopMutex) ); - *ppStatic = (lsm_mutex *)&aStaticNoopMutex[iMutex-1]; - return LSM_OK; -} -static int lsmWin32OsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - NoopMutex *p; - p = (NoopMutex *)lsmMallocZero(pEnv, sizeof(NoopMutex)); - if( p ) p->pEnv = pEnv; - *ppNew = (lsm_mutex *)p; - return (p ? LSM_OK : LSM_NOMEM_BKPT); -} -static void lsmWin32OsMutexDel(lsm_mutex *pMutex) { - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bStatic==0 && p->pEnv ); - lsmFree(p->pEnv, p); -} -static void lsmWin32OsMutexEnter(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; -} -static int lsmWin32OsMutexTry(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; - return 0; -} -static void lsmWin32OsMutexLeave(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==1 ); - p->bHeld = 0; -} -#ifdef LSM_DEBUG -static int lsmWin32OsMutexHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? p->bHeld : 1; -} -static int lsmWin32OsMutexNotHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? !p->bHeld : 1; -} -#endif -/***************************************************************************/ -#endif /* else LSM_MUTEX_NONE */ - -/* Without LSM_DEBUG, the MutexHeld tests are never called */ -#ifndef LSM_DEBUG -# define lsmWin32OsMutexHeld 0 -# define lsmWin32OsMutexNotHeld 0 -#endif - -lsm_env *lsm_default_env(void){ - static lsm_env win32_env = { - sizeof(lsm_env), /* nByte */ - 1, /* iVersion */ - /***** file i/o ******************/ - 0, /* pVfsCtx */ - lsmWin32OsFullpath, /* xFullpath */ - lsmWin32OsOpen, /* xOpen */ - lsmWin32OsRead, /* xRead */ - lsmWin32OsWrite, /* xWrite */ - lsmWin32OsTruncate, /* xTruncate */ - lsmWin32OsSync, /* xSync */ - lsmWin32OsSectorSize, /* xSectorSize */ - lsmWin32OsRemap, /* xRemap */ - lsmWin32OsFileid, /* xFileid */ - lsmWin32OsClose, /* xClose */ - lsmWin32OsUnlink, /* xUnlink */ - lsmWin32OsLock, /* xLock */ - lsmWin32OsTestLock, /* xTestLock */ - lsmWin32OsShmMap, /* xShmMap */ - lsmWin32OsShmBarrier, /* xShmBarrier */ - lsmWin32OsShmUnmap, /* xShmUnmap */ - /***** memory allocation *********/ - 0, /* pMemCtx */ - lsmWin32OsMalloc, /* xMalloc */ - lsmWin32OsRealloc, /* xRealloc */ - lsmWin32OsFree, /* xFree */ - lsmWin32OsMSize, /* xSize */ - /***** mutexes *********************/ - 0, /* pMutexCtx */ - lsmWin32OsMutexStatic, /* xMutexStatic */ - lsmWin32OsMutexNew, /* xMutexNew */ - lsmWin32OsMutexDel, /* xMutexDel */ - lsmWin32OsMutexEnter, /* xMutexEnter */ - lsmWin32OsMutexTry, /* xMutexTry */ - lsmWin32OsMutexLeave, /* xMutexLeave */ - lsmWin32OsMutexHeld, /* xMutexHeld */ - lsmWin32OsMutexNotHeld, /* xMutexNotHeld */ - /***** other *********************/ - lsmWin32OsSleep, /* xSleep */ - }; - return &win32_env; -} - -#endif diff --git a/ext/lsm1/test/lsm1_common.tcl b/ext/lsm1/test/lsm1_common.tcl deleted file mode 100644 index 0e6cd84e3..000000000 --- a/ext/lsm1/test/lsm1_common.tcl +++ /dev/null @@ -1,38 +0,0 @@ -# 2014 Dec 19 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# - -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. .. test] -} -source $testdir/tester.tcl - -# Check if the lsm1 extension has been compiled. -if {$::tcl_platform(platform) == "windows"} { - set lsm1 lsm.dll -} else { - set lsm1 lsm.so -} - -if {[file exists [file join .. $lsm1]]} { - proc return_if_no_lsm1 {} {} -} else { - proc return_if_no_lsm1 {} { - finish_test - return -code return - } - return -} - -proc load_lsm1_vtab {db} { - db enable_load_extension 1 - db eval {SELECT load_extension('../lsm')} -} diff --git a/ext/lsm1/test/lsm1_simple.test b/ext/lsm1/test/lsm1_simple.test deleted file mode 100644 index 2eab50a83..000000000 --- a/ext/lsm1/test/lsm1_simple.test +++ /dev/null @@ -1,152 +0,0 @@ -# 2017 July 14 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# This file implements regression tests for SQLite library. The -# focus of this script is testing the lsm1 virtual table module. -# - -source [file join [file dirname [info script]] lsm1_common.tcl] -set testprefix lsm1_simple -return_if_no_lsm1 -load_lsm1_vtab db - -forcedelete testlsm.db - -do_execsql_test 100 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,UINT,b,c,d); - PRAGMA table_info(x1); -} { - 0 a UINT 1 {} 1 - 1 b {} 0 {} 0 - 2 c {} 0 {} 0 - 3 d {} 0 {} 0 -} - -do_execsql_test 110 { - INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), - (12,NULL,3.25,-559281390); - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 33} -do_execsql_test 111 { - SELECT a, quote(lsm1_key), quote(lsm1_value) FROM x1; -} {8 X'08' X'2162616E6A6F1633323105' 12 X'0C' X'05320000000000000A401FFB42ABE9DB' 15 X'0F' X'4284C6'} - -do_execsql_test 120 { - UPDATE x1 SET d = d+1.0 WHERE a=15; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 34.0} - -do_execsql_test 130 { - UPDATE x1 SET a=123456789 WHERE a=12; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 15 11 22 34.0 123456789 NULL 3.25 -559281390} -do_execsql_test 131 { - SELECT quote(lsm1_key), printf('0x%x',a) FROM x1 WHERE a > 100000000; -} {X'FB075BCD15' 0x75bcd15} - -do_execsql_test 140 { - DELETE FROM x1 WHERE a=15; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 123456789 NULL 3.25 -559281390} - -do_test 150 { - lsort [glob testlsm.db*] -} {testlsm.db testlsm.db-log testlsm.db-shm} - -db close -do_test 160 { - lsort [glob testlsm.db*] -} {testlsm.db} - -forcedelete testlsm.db -forcedelete test.db -sqlite3 db test.db -load_lsm1_vtab db - - -do_execsql_test 200 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); - PRAGMA table_info(x1); -} { - 0 a TEXT 1 {} 1 - 1 b {} 0 {} 0 - 2 c {} 0 {} 0 - 3 d {} 0 {} 0 -} -do_execsql_test 210 { - INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), - (12,NULL,3.25,-559281390); - SELECT quote(a), quote(b), quote(c), quote(d), '|' FROM x1; -} {'12' NULL 3.25 -559281390 | '15' 11 22 33 | '8' 'banjo' X'333231' NULL |} -do_execsql_test 211 { - SELECT quote(a), quote(lsm1_key), quote(lsm1_value), '|' FROM x1; -} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB' | '15' X'3135' X'4284C6' | '8' X'38' X'2162616E6A6F1633323105' |} -do_execsql_test 212 { - SELECT quote(a), quote(lsm1_key), quote(lsm1_value) FROM x1 WHERE a='12'; -} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB'} - -#------------------------------------------------------------------------- -reset_db -forcedelete testlsm.db -load_lsm1_vtab db -do_execsql_test 300 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); -} -do_eqp_test 310 { - SELECT * FROM x1 WHERE a=? -} {SCAN TABLE x1 VIRTUAL TABLE INDEX 0:} - -do_eqp_test 320 { - SELECT * FROM x1 WHERE a>? -} {SCAN TABLE x1 VIRTUAL TABLE INDEX 2:} - -do_eqp_test 330 { - SELECT * FROM x1 WHERE a<? -} {SCAN TABLE x1 VIRTUAL TABLE INDEX 3:} -do_eqp_test 340 { - SELECT * FROM x1 WHERE a BETWEEN ? AND ? -} {SCAN TABLE x1 VIRTUAL TABLE INDEX 1:} - -#------------------------------------------------------------------------- -reset_db -forcedelete testlsm.db -load_lsm1_vtab db -do_execsql_test 400 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b); - INSERT INTO x1 VALUES('one', 1); - INSERT INTO x1 VALUES('two', 2); - INSERT INTO x1 VALUES('three', 3); - INSERT INTO x1 VALUES('four', 4); - INSERT INTO x1 VALUES('five', 5); -} -do_execsql_test 410 { - SELECT b FROM x1 WHERE a = 'two' -} {2} -do_execsql_test 411 { - SELECT b FROM x1 WHERE a = 'one' -} {1} -do_execsql_test 412 { - SELECT b FROM x1 WHERE a = 'five' -} {5} - -do_execsql_test 420 { - SELECT b FROM x1 WHERE a BETWEEN 'one' AND 'three'; -} {1 3} -do_execsql_test 421 { - SELECT b FROM x1 WHERE a BETWEEN 'five' AND 'two'; -} {5 4 1 3 2} -do_execsql_test 421 { - SELECT b FROM x1 WHERE a > 'five'; -} {4 1 3 2} -do_execsql_test 421 { - SELECT b FROM x1 WHERE a <= 'three'; -} {3 1 4 5} - -finish_test diff --git a/ext/lsm1/tool/mklsm1c.tcl b/ext/lsm1/tool/mklsm1c.tcl deleted file mode 100644 index d4a317b70..000000000 --- a/ext/lsm1/tool/mklsm1c.tcl +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/sh -# restart with tclsh \ -exec tclsh "$0" "$@" - -set srcdir [file dirname [file dirname [info script]]] -set G(src) [string map [list %dir% $srcdir] { - %dir%/lsm.h - %dir%/lsmInt.h - %dir%/lsm_vtab.c - %dir%/lsm_ckpt.c - %dir%/lsm_file.c - %dir%/lsm_log.c - %dir%/lsm_main.c - %dir%/lsm_mem.c - %dir%/lsm_mutex.c - %dir%/lsm_shared.c - %dir%/lsm_sorted.c - %dir%/lsm_str.c - %dir%/lsm_tree.c - %dir%/lsm_unix.c - %dir%/lsm_varint.c - %dir%/lsm_win32.c -}] - -set G(hdr) { - -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif -#if defined(NDEBUG) && defined(SQLITE_DEBUG) -# undef NDEBUG -#endif - -} - -set G(footer) { - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */ -} - -#------------------------------------------------------------------------- -# Read and return the entire contents of text file $zFile from disk. -# -proc readfile {zFile} { - set fd [open $zFile] - set data [read $fd] - close $fd - return $data -} - -proc lsm1c_init {zOut} { - global G - set G(fd) stdout - set G(fd) [open $zOut w] - - puts -nonewline $G(fd) $G(hdr) -} - -proc lsm1c_printfile {zIn} { - global G - set data [readfile $zIn] - set zTail [file tail $zIn] - puts $G(fd) "#line 1 \"$zTail\"" - - foreach line [split $data "\n"] { - if {[regexp {^# *include.*lsm} $line]} { - set line "/* $line */" - } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } { - set line "static $line" - } - puts $G(fd) $line - } -} - -proc lsm1c_close {} { - global G - puts -nonewline $G(fd) $G(footer) - if {$G(fd)!="stdout"} { - close $G(fd) - } -} - - -lsm1c_init lsm1.c -foreach f $G(src) { lsm1c_printfile $f } -lsm1c_close diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 01d9c220f..22d61df61 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -899,9 +899,9 @@ static int fsdirFilter( ){ const char *zDir = 0; fsdir_cursor *pCur = (fsdir_cursor*)cur; + int i; (void)idxStr; fsdirResetCursor(pCur); - int i; if( idxNum==0 ){ fsdirSetErrmsg(pCur, "table function fsdir requires an argument"); diff --git a/ext/misc/vtablog.c b/ext/misc/vtablog.c index e8f084e1b..44acc32e6 100644 --- a/ext/misc/vtablog.c +++ b/ext/misc/vtablog.c @@ -14,6 +14,13 @@ ** on stdout when its key interfaces are called. This is intended for ** interactive analysis and debugging of virtual table interfaces. ** +** To build this extension as a separately loaded shared library or +** DLL, use compiler command-lines similar to the following: +** +** (linux) gcc -fPIC -shared vtablog.c -o vtablog.so +** (mac) clang -fPIC -dynamiclib vtablog.c -o vtablog.dylib +** (windows) cl vtablog.c -link -dll -out:vtablog.dll +** ** Usage example: ** ** .load ./vtablog @@ -436,6 +443,39 @@ static int vtablogFilter( } /* +** Return an sqlite3_index_info operator name in static space. +** The name is possibly overwritten on subsequent calls. +*/ +static char *vtablogOpName(unsigned char op){ + static char zUnknown[30]; + char *zOut; + switch( op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: zOut = "EQ"; break; + case SQLITE_INDEX_CONSTRAINT_GT: zOut = "GT"; break; + case SQLITE_INDEX_CONSTRAINT_LE: zOut = "LE"; break; + case SQLITE_INDEX_CONSTRAINT_LT: zOut = "LT"; break; + case SQLITE_INDEX_CONSTRAINT_GE: zOut = "GE"; break; + case SQLITE_INDEX_CONSTRAINT_MATCH: zOut = "MATCH"; break; + case SQLITE_INDEX_CONSTRAINT_LIKE: zOut = "LIKE"; break; + case SQLITE_INDEX_CONSTRAINT_GLOB: zOut = "GLOB"; break; + case SQLITE_INDEX_CONSTRAINT_REGEXP: zOut = "REGEXP"; break; + case SQLITE_INDEX_CONSTRAINT_NE: zOut = "NE"; break; + case SQLITE_INDEX_CONSTRAINT_ISNOT: zOut = "ISNOT"; break; + case SQLITE_INDEX_CONSTRAINT_ISNOTNULL: zOut = "ISNOTNULL"; break; + case SQLITE_INDEX_CONSTRAINT_ISNULL: zOut = "ISNULL"; break; + case SQLITE_INDEX_CONSTRAINT_IS: zOut = "IS"; break; + case SQLITE_INDEX_CONSTRAINT_LIMIT: zOut = "LIMIT"; break; + case SQLITE_INDEX_CONSTRAINT_OFFSET: zOut = "OFFSET"; break; + case SQLITE_INDEX_CONSTRAINT_FUNCTION: zOut = "FUNCTION"; break; + default: + sqlite3_snprintf(sizeof(zUnknown),zUnknown,"%d",op); + zOut = zUnknown; + break; + } + return zOut; +} + +/* ** SQLite will invoke this method one or more times while planning a query ** that uses the vtablog virtual table. This routine needs to create ** a query plan for each invocation and compute an estimated cost for that @@ -451,14 +491,23 @@ static int vtablogBestIndex( printf(" colUsed: 0x%016llx\n", p->colUsed); printf(" nConstraint: %d\n", p->nConstraint); for(i=0; i<p->nConstraint; i++){ + sqlite3_value *pVal = 0; + int rc = sqlite3_vtab_rhs_value(p, i, &pVal); printf( - " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", + " constraint[%d]: col=%d termid=%d op=%s usabled=%d coll=%s rhs=", i, p->aConstraint[i].iColumn, p->aConstraint[i].iTermOffset, - p->aConstraint[i].op, + vtablogOpName(p->aConstraint[i].op), p->aConstraint[i].usable, - sqlite3_vtab_collation(p,i)); + sqlite3_vtab_collation(p,i) + ); + if( rc==SQLITE_OK ){ + vtablogQuote(pVal); + printf("\n"); + }else{ + printf("N/A\n"); + } } printf(" nOrderBy: %d\n", p->nOrderBy); for(i=0; i<p->nOrderBy; i++){ diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 2377457df..9e78e7230 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -116,6 +116,7 @@ static const char ZIPFILE_SCHEMA[] = #define ZIPFILE_F_COLUMN_IDX 7 /* Index of column "file" in the above */ #define ZIPFILE_BUFFER_SIZE (64*1024) +#define ZIPFILE_MX_NAME (250) /* Windows limitation on filename size */ /* @@ -672,6 +673,7 @@ static int zipfileReadLFH( pLFH->szUncompressed = zipfileRead32(aRead); pLFH->nFile = zipfileRead16(aRead); pLFH->nExtra = zipfileRead16(aRead); + if( pLFH->nFile>ZIPFILE_MX_NAME ) rc = SQLITE_ERROR; } return rc; } @@ -885,8 +887,12 @@ static int zipfileGetEntry( pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; pNew->iDataOff += lfh.nFile + lfh.nExtra; if( aBlob && pNew->cds.szCompressed ){ - pNew->aData = &pNew->aExtra[nExtra]; - memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + if( pNew->iDataOff + pNew->cds.szCompressed > nBlob ){ + rc = SQLITE_CORRUPT; + }else{ + pNew->aData = &pNew->aExtra[nExtra]; + memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + } } }else{ *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", @@ -1673,6 +1679,11 @@ static int zipfileUpdate( zPath = (const char*)sqlite3_value_text(apVal[2]); if( zPath==0 ) zPath = ""; nPath = (int)strlen(zPath); + if( nPath>ZIPFILE_MX_NAME ){ + zipfileTableErr(pTab, "filename too long; max: %d bytes", + ZIPFILE_MX_NAME); + rc = SQLITE_CONSTRAINT; + } mTime = zipfileGetTime(apVal[4]); } @@ -2034,6 +2045,13 @@ static void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ rc = SQLITE_ERROR; goto zipfile_step_out; } + if( nName>ZIPFILE_MX_NAME ){ + zErr = sqlite3_mprintf( + "filename argument to zipfile() too big; max: %d bytes", + ZIPFILE_MX_NAME); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use ** deflate compression) or NULL (choose automatically). */ diff --git a/ext/misc/zorder.c b/ext/misc/zorder.c index c385d3c3c..c4c5fcdc7 100644 --- a/ext/misc/zorder.c +++ b/ext/misc/zorder.c @@ -16,6 +16,17 @@ ** ** unzorder(Z,N,I) Extract the I-th dimension from N-dimensional ** Morton code Z. +** +** Compiling: +** +** (linux) gcc -fPIC -shared zorder.c -o zorder.so +** (mac) clang -fPIC -dynamiclib zorder.c -o zorder.dylib +** (windows) cl zorder.c -link -dll -out:zorder.dll +** +** Usage example: +** +** .load ./zorder +** SELECT zorder(1,2,3,4); */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -25,43 +36,53 @@ SQLITE_EXTENSION_INIT1 /* ** Functions: zorder(X0,X1,....) ** -** Convert integers X0, X1, ... into morton code. +** Convert integers X0, X1, ... into morton code. There must be at least +** two arguments. There may be no more than 24 arguments. ** -** The output is a signed 64-bit integer. If any argument is too large, -** an error is thrown. +** The output is a signed 64-bit integer. If any argument is too large +** to be successfully encoded into a morton code, an error is raised. */ static void zorderFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ - sqlite3_int64 z, x[63]; + sqlite3_int64 z, x[24]; int i, j; z = 0; + if( argc<2 || argc>24 ){ + sqlite3_result_error(context, + "zorder() needs between 2 and 24 arguments4", -1); + return; + } for(i=0; i<argc; i++){ x[i] = sqlite3_value_int64(argv[i]); } - if( argc>0 ){ - for(i=0; i<63; i++){ - j = i%argc; - z |= (x[j]&1)<<i; - x[j] >>= 1; - } + for(i=0; i<63; i++){ + j = i%argc; + z |= (x[j]&1)<<i; + x[j] >>= 1; } sqlite3_result_int64(context, z); for(i=0; i<argc; i++){ if( x[i] ){ - sqlite3_result_error(context, "parameter too large", -1); + char *z = sqlite3_mprintf( + "the %r argument to zorder() (%lld) is too large " + "for a 64-bit %d-dimensional Morton code", + i+1, sqlite3_value_int64(argv[i]), argc); + sqlite3_result_error(context, z, -1); + sqlite3_free(z); + break; } } } /* -** Functions: unzorder(Z,N,I) +** Function: unzorder(Z,N,K) ** -** Assuming that Z is an N-dimensional Morton code, extract the I-th -** dimension. +** Assuming that Z is an N-dimensional Morton code, extract the K-th +** dimension. K is between 0 and N-1. N must be between 2 and 24. */ static void unzorderFunc( sqlite3_context *context, @@ -72,7 +93,18 @@ static void unzorderFunc( int j, k; z = sqlite3_value_int64(argv[0]); n = sqlite3_value_int64(argv[1]); + if( n<2 || n>24 ){ + sqlite3_result_error(context, + "N argument to unzorder(Z,N,K) should be between 2 and 24", + -1); + return; + } i = sqlite3_value_int64(argv[2]); + if( i<0 || i>=n ){ + sqlite3_result_error(context, + "K argument to unzorder(Z,N,K) should be between 0 and N-1", -1); + return; + } x = 0; for(k=0, j=i; j<63; j+=n, k++){ x |= ((z>>j)&1)<<k; diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index f7d3bda01..fb35bc10e 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1135,6 +1135,12 @@ static void resetCursor(RtreeCursor *pCsr){ pCsr->base.pVtab = (sqlite3_vtab*)pRtree; pCsr->pReadAux = pStmt; + /* The following will only fail if the previous sqlite3_step() call failed, + ** in which case the error has already been caught. This statement never + ** encounters an error within an sqlite3_column_xxx() function, as it + ** calls sqlite3_column_value(), which does not use malloc(). So it is safe + ** to ignore the error code here. */ + sqlite3_reset(pStmt); } /* diff --git a/ext/rtree/rtreeH.test b/ext/rtree/rtreeH.test index e26107f07..79bf9808d 100644 --- a/ext/rtree/rtreeH.test +++ b/ext/rtree/rtreeH.test @@ -99,5 +99,24 @@ do_execsql_test rtreeH-300 { ORDER BY id; } {box-48,48 box-49,48 box-48,49 xbox-49,49} +#------------------------------------------------------------------------- +reset_db +do_execsql_test rtreeH-300 { + CREATE TABLE t0(c0); + INSERT INTO t0(c0) VALUES (NULL); + INSERT INTO t0(c0) VALUES (1); + CREATE VIRTUAL TABLE t1 USING rtree(c0, c1, c2, +c3 BLOB ); + INSERT INTO t1(c2, c3, c0) VALUES (1, 2, 1); +} + +do_execsql_test rtreeH-310 { + SELECT 1 FROM t1 WHERE t1.c3; +} {1} + +do_execsql_test rtreeH-320 { + SELECT * FROM t0 WHERE NOT EXISTS ( + SELECT 1 FROM t1 WHERE t1.c3 OR t0.c0 ISNULL + ); +} {} finish_test diff --git a/ext/session/sessionI.test b/ext/session/sessionI.test new file mode 100644 index 000000000..2012f24dc --- /dev/null +++ b/ext/session/sessionI.test @@ -0,0 +1,88 @@ +# 2015 July 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionI + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 1.0 { + do_common_sql { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + } +} {} + +set C [changeset_from_sql { + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t1 VALUES(4, 'four'); + INSERT INTO t1 VALUES(5, 'five'); + INSERT INTO t1 VALUES(6, 'six'); +}] + +do_execsql_test 1.1 { + SELECT * FROM t1 +} { + 1 one 2 two 3 three 4 four 5 five 6 six +} + +proc xFilter {data} { + foreach {op tname flag pk old new} $data {} + if {$op=="INSERT"} { + set ipk [lindex $new 1] + return [expr $ipk % 2] + } + return 1 +} +proc xConflict {args} { +} + +sqlite3changeset_apply_v3 db2 $C xConflict xFilter + +do_execsql_test -db db2 1.2 { + SELECT * FROM t1 +} { + 1 one 3 three 5 five +} + +do_execsql_test -db db2 1.3 { + DELETE FROM t1 +} +sqlite3changeset_apply_v3 db2 $C xConflict + +do_execsql_test -db db2 1.4 { + SELECT * FROM t1 +} { + 1 one 2 two 3 three 4 four 5 five 6 six +} + +proc xFilter2 {data} { + return 0 +} +do_execsql_test -db db2 1.5 { + DELETE FROM t1 +} +sqlite3changeset_apply_v3 db2 $C xConflict xFilter2 +do_execsql_test -db db2 1.6 { + SELECT * FROM t1 +} { } + +finish_test diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 175cacbe8..df40fdc1c 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -5182,6 +5182,10 @@ static int sessionChangesetApply( void *pCtx, /* Copy of sixth arg to _apply() */ const char *zTab /* Table name */ ), + int(*xFilterIter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), int(*xConflict)( void *pCtx, /* Copy of fifth arg to _apply() */ int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ @@ -5322,6 +5326,9 @@ static int sessionChangesetApply( ** next change. A log message has already been issued. */ if( schemaMismatch ) continue; + /* If this is a call to apply_v3(), invoke xFilterIter here. */ + if( xFilterIter && 0==xFilterIter(pCtx, pIter) ) continue; + rc = sessionApplyOneWithRetry(db, pIter, &sApply, xConflict, pCtx); } @@ -5390,17 +5397,64 @@ static int sessionChangesetApply( } /* -** Apply the changeset passed via pChangeset/nChangeset to the main -** database attached to handle "db". +** This function is called by all six sqlite3changeset_apply() variants: +** +** + sqlite3changeset_apply() +** + sqlite3changeset_apply_v2() +** + sqlite3changeset_apply_v3() +** + sqlite3changeset_apply_strm() +** + sqlite3changeset_apply_strm_v2() +** + sqlite3changeset_apply_strm_v3() +** +** Arguments passed to this function are as follows: +** +** db: +** Database handle to apply changeset to main database of. +** +** nChangeset/pChangeset: +** These are both passed zero for the streaming variants. For the normal +** apply() functions, these are passed the size of and the buffer containing +** the changeset, respectively. +** +** xInput/pIn: +** These are both passed zero for the normal variants. For the streaming +** apply() functions, these are passed the input callback and context +** pointer, respectively. +** +** xFilter: +** The filter function as passed to apply() or apply_v2() (to filter by +** table name), if any. This is always NULL for apply_v3() calls. +** +** xFilterIter: +** The filter function as passed to apply_v3(), if any. +** +** xConflict: +** The conflict handler callback (must not be NULL). +** +** pCtx: +** The context pointer passed to the xFilter and xConflict handler callbacks. +** +** ppRebase, pnRebase: +** Zero for apply(). The rebase changeset output pointers, if any, for +** apply_v2() and apply_v3(). +** +** flags: +** Zero for apply(). The flags parameter for apply_v2() and apply_v3(). */ -int sqlite3changeset_apply_v2( +static int sessionChangesetApplyV23( sqlite3 *db, /* Apply change to "main" db of this handle */ int nChangeset, /* Size of changeset in bytes */ void *pChangeset, /* Changeset blob */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ int(*xFilter)( void *pCtx, /* Copy of sixth arg to _apply() */ const char *zTab /* Table name */ ), + int(*xFilterIter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing current change */ + ), int(*xConflict)( void *pCtx, /* Copy of sixth arg to _apply() */ int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ @@ -5411,19 +5465,75 @@ int sqlite3changeset_apply_v2( int flags ){ sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ - int bInv = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); - int rc = sessionChangesetStart(&pIter, 0, 0, nChangeset, pChangeset, bInv, 1); - + int bInverse = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + int rc = sessionChangesetStart( + &pIter, xInput, pIn, nChangeset, pChangeset, bInverse, 1 + ); if( rc==SQLITE_OK ){ - rc = sessionChangesetApply( - db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags + rc = sessionChangesetApply(db, pIter, + xFilter, xFilterIter, xConflict, pCtx, ppRebase, pnRebase, flags ); } - return rc; } /* +** Apply the changeset passed via pChangeset/nChangeset to the main +** database attached to handle "db". +*/ +int sqlite3changeset_apply_v2( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + xFilter, 0, xConflict, pCtx, + ppRebase, pnRebase, flags + ); +} + +/* +** Apply the changeset passed via pChangeset/nChangeset to the main +** database attached to handle "db". +*/ +int sqlite3changeset_apply_v3( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing current change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + 0, xFilter, xConflict, pCtx, + ppRebase, pnRebase, flags + ); +} + +/* ** Apply the changeset passed via pChangeset/nChangeset to the main database ** attached to handle "db". Invoke the supplied conflict handler callback ** to resolve any conflicts encountered while applying the change. @@ -5443,8 +5553,10 @@ int sqlite3changeset_apply( ), void *pCtx /* First argument passed to xConflict */ ){ - return sqlite3changeset_apply_v2( - db, nChangeset, pChangeset, xFilter, xConflict, pCtx, 0, 0, 0 + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + xFilter, 0, xConflict, pCtx, + 0, 0, 0 ); } @@ -5453,6 +5565,29 @@ int sqlite3changeset_apply( ** attached to handle "db". Invoke the supplied conflict handler callback ** to resolve any conflicts encountered while applying the change. */ +int sqlite3changeset_apply_v3_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + 0, xFilter, xConflict, pCtx, + ppRebase, pnRebase, flags + ); +} int sqlite3changeset_apply_v2_strm( sqlite3 *db, /* Apply change to "main" db of this handle */ int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ @@ -5470,15 +5605,11 @@ int sqlite3changeset_apply_v2_strm( void **ppRebase, int *pnRebase, int flags ){ - sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ - int bInverse = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); - int rc = sessionChangesetStart(&pIter, xInput, pIn, 0, 0, bInverse, 1); - if( rc==SQLITE_OK ){ - rc = sessionChangesetApply( - db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags - ); - } - return rc; + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + xFilter, 0, xConflict, pCtx, + ppRebase, pnRebase, flags + ); } int sqlite3changeset_apply_strm( sqlite3 *db, /* Apply change to "main" db of this handle */ @@ -5495,8 +5626,10 @@ int sqlite3changeset_apply_strm( ), void *pCtx /* First argument passed to xConflict */ ){ - return sqlite3changeset_apply_v2_strm( - db, xInput, pIn, xFilter, xConflict, pCtx, 0, 0, 0 + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + xFilter, 0, xConflict, pCtx, + 0, 0, 0 ); } diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index 4dff5ce87..a3b6987b9 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -1115,13 +1115,22 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** the changeset passed via the second and third arguments. ** ** The fourth argument (xFilter) passed to these functions is the "filter -** callback". If it is not NULL, then for each table affected by at least one -** change in the changeset, the filter callback is invoked with -** the table name as the second argument, and a copy of the context pointer -** passed as the sixth argument as the first. If the "filter callback" -** returns zero, then no attempt is made to apply any changes to the table. -** Otherwise, if the return value is non-zero or the xFilter argument to -** is NULL, all changes related to the table are attempted. +** callback". This may be passed NULL, in which case all changes in the +** changeset are applied to the database. For sqlite3changeset_apply() and +** sqlite3_changeset_apply_v2(), if it is not NULL, then it is invoked once +** for each table affected by at least one change in the changeset. In this +** case the table name is passed as the second argument, and a copy of +** the context pointer passed as the sixth argument to apply() or apply_v2() +** as the first. If the "filter callback" returns zero, then no attempt is +** made to apply any changes to the table. Otherwise, if the return value is +** non-zero, all changes related to the table are attempted. +** +** For sqlite3_changeset_apply_v3(), the xFilter callback is invoked once +** per change. The second argument in this case is an sqlite3_changeset_iter +** that may be queried using the usual APIs for the details of the current +** change. If the "filter callback" returns zero in this case, then no attempt +** is made to apply the current change. If it returns non-zero, the change +** is applied. ** ** For each table that is not excluded by the filter callback, this function ** tests that the target database contains a compatible table. A table is @@ -1142,11 +1151,11 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** one such warning is issued for each table in the changeset. ** ** For each change for which there is a compatible table, an attempt is made -** to modify the table contents according to the UPDATE, INSERT or DELETE -** change. If a change cannot be applied cleanly, the conflict handler -** function passed as the fifth argument to sqlite3changeset_apply() may be -** invoked. A description of exactly when the conflict handler is invoked for -** each type of change is below. +** to modify the table contents according to each UPDATE, INSERT or DELETE +** change that is not excluded by a filter callback. If a change cannot be +** applied cleanly, the conflict handler function passed as the fifth argument +** to sqlite3changeset_apply() may be invoked. A description of exactly when +** the conflict handler is invoked for each type of change is below. ** ** Unlike the xFilter argument, xConflict may not be passed NULL. The results ** of passing anything other than a valid function pointer as the xConflict @@ -1297,6 +1306,23 @@ int sqlite3changeset_apply_v2( void **ppRebase, int *pnRebase, /* OUT: Rebase data */ int flags /* SESSION_CHANGESETAPPLY_* flags */ ); +int sqlite3changeset_apply_v3( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); /* ** CAPI3REF: Flags for sqlite3changeset_apply_v2 @@ -1716,6 +1742,23 @@ int sqlite3changeset_apply_v2_strm( void **ppRebase, int *pnRebase, int flags ); +int sqlite3changeset_apply_v3_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); int sqlite3changeset_concat_strm( int (*xInputA)(void *pIn, void *pData, int *pnData), void *pInA, diff --git a/ext/session/test_session.c b/ext/session/test_session.c index f28604abc..6ad5b3774 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -520,6 +520,65 @@ static int test_obj_eq_string(Tcl_Obj *p, const char *z){ return (nObj==n && (n==0 || 0==memcmp(zObj, z, n))); } +static Tcl_Obj *testIterData(sqlite3_changeset_iter *pIter){ + Tcl_Obj *pVar = 0; + int nCol; /* Number of columns in table */ + int nCol2; /* Number of columns in table */ + int op; /* SQLITE_INSERT, UPDATE or DELETE */ + const char *zTab; /* Name of table change applies to */ + Tcl_Obj *pOld; /* Vector of old.* values */ + Tcl_Obj *pNew; /* Vector of new.* values */ + int bIndirect; + + char *zPK; + unsigned char *abPK; + int i; + + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + pVar = Tcl_NewObj(); + + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj( + op==SQLITE_INSERT ? "INSERT" : + op==SQLITE_UPDATE ? "UPDATE" : + "DELETE", -1 + )); + + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zTab, -1)); + Tcl_ListObjAppendElement(0, pVar, Tcl_NewBooleanObj(bIndirect)); + + zPK = ckalloc(nCol+1); + memset(zPK, 0, nCol+1); + sqlite3changeset_pk(pIter, &abPK, &nCol2); + assert( nCol==nCol2 ); + for(i=0; i<nCol; i++){ + zPK[i] = (abPK[i] ? 'X' : '.'); + } + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zPK, -1)); + ckfree(zPK); + + pOld = Tcl_NewObj(); + if( op!=SQLITE_INSERT ){ + for(i=0; i<nCol; i++){ + sqlite3_value *pVal; + sqlite3changeset_old(pIter, i, &pVal); + test_append_value(pOld, pVal); + } + } + pNew = Tcl_NewObj(); + if( op!=SQLITE_DELETE ){ + for(i=0; i<nCol; i++){ + sqlite3_value *pVal; + sqlite3changeset_new(pIter, i, &pVal); + test_append_value(pNew, pVal); + } + } + Tcl_ListObjAppendElement(0, pVar, pOld); + Tcl_ListObjAppendElement(0, pVar, pNew); + + return pVar; +} + + static int test_filter_handler( void *pCtx, /* Pointer to TestConflictHandler structure */ const char *zTab /* Table name */ @@ -543,6 +602,29 @@ static int test_filter_handler( return res; } +static int test_filter_v3_handler( + void *pCtx, /* Pointer to TestConflictHandler structure */ + sqlite3_changeset_iter *pIter +){ + TestConflictHandler *p = (TestConflictHandler *)pCtx; + int res = 1; + Tcl_Obj *pEval = 0; + Tcl_Interp *interp = p->interp; + + pEval = Tcl_DuplicateObj(p->pFilterScript); + Tcl_IncrRefCount(pEval); + Tcl_ListObjAppendElement(0, pEval, testIterData(pIter)); + + if( TCL_OK!=Tcl_EvalObjEx(interp, pEval, TCL_EVAL_GLOBAL) + || TCL_OK!=Tcl_GetIntFromObj(interp, Tcl_GetObjResult(interp), &res) + ){ + Tcl_BackgroundError(interp); + } + + Tcl_DecrRefCount(pEval); + return res; +} + static int test_conflict_handler( void *pCtx, /* Pointer to TestConflictHandler structure */ int eConf, /* DATA, MISSING, CONFLICT, CONSTRAINT */ @@ -776,7 +858,7 @@ static int testStreamInput( static int SQLITE_TCLAPI testSqlite3changesetApply( - int bV2, + int iVersion, void * clientData, Tcl_Interp *interp, int objc, @@ -793,11 +875,13 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( int nRebase = 0; int flags = 0; /* Flags for apply_v2() */ + assert( iVersion==1 || iVersion==2 || iVersion==3 ); + memset(&sStr, 0, sizeof(sStr)); sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); /* Check for the -nosavepoint, -invert or -ignorenoop switches */ - if( bV2 ){ + if( iVersion==2 || iVersion==3 ){ while( objc>1 ){ const char *z1 = Tcl_GetString(objv[1]); int n = (int)strlen(z1); @@ -822,7 +906,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( if( objc!=4 && objc!=5 ){ const char *zMsg; - if( bV2 ){ + if( iVersion==2 || iVersion==3 ){ zMsg = "?-nosavepoint? ?-inverse? ?-ignorenoop? " "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?"; }else{ @@ -842,30 +926,49 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( ctx.interp = interp; if( sStr.nStream==0 ){ - if( bV2==0 ){ - rc = sqlite3changeset_apply(db, (int)nChangeset, pChangeset, - (objc==5)?test_filter_handler:0, test_conflict_handler, (void *)&ctx - ); - }else{ - rc = sqlite3changeset_apply_v2(db, (int)nChangeset, pChangeset, - (objc==5)?test_filter_handler:0, test_conflict_handler, (void *)&ctx, - &pRebase, &nRebase, flags - ); + switch( iVersion ){ + case 1: + rc = sqlite3changeset_apply(db, (int)nChangeset, pChangeset, + (objc==5)?test_filter_handler:0, test_conflict_handler, (void*)&ctx + ); + break; + case 2: + rc = sqlite3changeset_apply_v2(db, (int)nChangeset, pChangeset, + (objc==5)?test_filter_handler:0, test_conflict_handler, (void*)&ctx, + &pRebase, &nRebase, flags + ); + break; + case 3: + rc = sqlite3changeset_apply_v3(db, (int)nChangeset, pChangeset, + (objc==5)?test_filter_v3_handler:0, test_conflict_handler, + (void*)&ctx, &pRebase, &nRebase, flags + ); + break; } }else{ sStr.aData = (unsigned char*)pChangeset; sStr.nData = (int)nChangeset; - if( bV2==0 ){ - rc = sqlite3changeset_apply_strm(db, testStreamInput, (void*)&sStr, - (objc==5) ? test_filter_handler : 0, - test_conflict_handler, (void *)&ctx - ); - }else{ - rc = sqlite3changeset_apply_v2_strm(db, testStreamInput, (void*)&sStr, - (objc==5) ? test_filter_handler : 0, - test_conflict_handler, (void *)&ctx, - &pRebase, &nRebase, flags - ); + switch( iVersion ){ + case 1: + rc = sqlite3changeset_apply_strm(db, testStreamInput, (void*)&sStr, + (objc==5) ? test_filter_handler : 0, + test_conflict_handler, (void *)&ctx + ); + break; + case 2: + rc = sqlite3changeset_apply_v2_strm(db, testStreamInput, (void*)&sStr, + (objc==5) ? test_filter_handler : 0, + test_conflict_handler, (void *)&ctx, + &pRebase, &nRebase, flags + ); + break; + case 3: + rc = sqlite3changeset_apply_v3_strm(db, testStreamInput, (void*)&sStr, + (objc==5) ? test_filter_v3_handler : 0, + test_conflict_handler, (void *)&ctx, + &pRebase, &nRebase, flags + ); + break; } } @@ -873,7 +976,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( return test_session_error(interp, rc, 0); }else{ Tcl_ResetResult(interp); - if( bV2 && pRebase ){ + if( (iVersion==2 || iVersion==3) && pRebase ){ Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pRebase, nRebase)); } } @@ -890,7 +993,7 @@ static int SQLITE_TCLAPI test_sqlite3changeset_apply( int objc, Tcl_Obj *CONST objv[] ){ - return testSqlite3changesetApply(0, clientData, interp, objc, objv); + return testSqlite3changesetApply(1, clientData, interp, objc, objv); } /* ** sqlite3changeset_apply_v2 DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT? @@ -901,7 +1004,18 @@ static int SQLITE_TCLAPI test_sqlite3changeset_apply_v2( int objc, Tcl_Obj *CONST objv[] ){ - return testSqlite3changesetApply(1, clientData, interp, objc, objv); + return testSqlite3changesetApply(2, clientData, interp, objc, objv); +} +/* +** sqlite3changeset_apply_v3 DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT? +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_apply_v3( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + return testSqlite3changesetApply(3, clientData, interp, objc, objv); } /* @@ -1034,64 +1148,6 @@ static int SQLITE_TCLAPI test_sqlite3changeset_concat( return rc; } -static Tcl_Obj *testIterData(sqlite3_changeset_iter *pIter){ - Tcl_Obj *pVar = 0; - int nCol; /* Number of columns in table */ - int nCol2; /* Number of columns in table */ - int op; /* SQLITE_INSERT, UPDATE or DELETE */ - const char *zTab; /* Name of table change applies to */ - Tcl_Obj *pOld; /* Vector of old.* values */ - Tcl_Obj *pNew; /* Vector of new.* values */ - int bIndirect; - - char *zPK; - unsigned char *abPK; - int i; - - sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); - pVar = Tcl_NewObj(); - - Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj( - op==SQLITE_INSERT ? "INSERT" : - op==SQLITE_UPDATE ? "UPDATE" : - "DELETE", -1 - )); - - Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zTab, -1)); - Tcl_ListObjAppendElement(0, pVar, Tcl_NewBooleanObj(bIndirect)); - - zPK = ckalloc(nCol+1); - memset(zPK, 0, nCol+1); - sqlite3changeset_pk(pIter, &abPK, &nCol2); - assert( nCol==nCol2 ); - for(i=0; i<nCol; i++){ - zPK[i] = (abPK[i] ? 'X' : '.'); - } - Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zPK, -1)); - ckfree(zPK); - - pOld = Tcl_NewObj(); - if( op!=SQLITE_INSERT ){ - for(i=0; i<nCol; i++){ - sqlite3_value *pVal; - sqlite3changeset_old(pIter, i, &pVal); - test_append_value(pOld, pVal); - } - } - pNew = Tcl_NewObj(); - if( op!=SQLITE_DELETE ){ - for(i=0; i<nCol; i++){ - sqlite3_value *pVal; - sqlite3changeset_new(pIter, i, &pVal); - test_append_value(pNew, pVal); - } - } - Tcl_ListObjAppendElement(0, pVar, pOld); - Tcl_ListObjAppendElement(0, pVar, pNew); - - return pVar; -} - /* ** sqlite3session_foreach VARNAME CHANGESET SCRIPT */ @@ -1751,6 +1807,7 @@ int TestSession_Init(Tcl_Interp *interp){ { "sqlite3changeset_concat", test_sqlite3changeset_concat }, { "sqlite3changeset_apply", test_sqlite3changeset_apply }, { "sqlite3changeset_apply_v2", test_sqlite3changeset_apply_v2 }, + { "sqlite3changeset_apply_v3", test_sqlite3changeset_apply_v3 }, { "sqlite3changeset_apply_replace_all", test_sqlite3changeset_apply_replace_all }, { "sql_exec_changeset", test_sql_exec_changeset }, diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 28387872f..5cd0aa66a 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -39,10 +39,11 @@ ######################################################################## default: all #default: quick -MAKEFILE := $(lastword $(MAKEFILE_LIST)) -CLEAN_FILES := -DISTCLEAN_FILES := config.make -MAKING_CLEAN := $(if $(filter %clean,$(MAKECMDGOALS)),1,0) +MAKEFILE = $(lastword $(MAKEFILE_LIST)) +MAKEFILE.fiddle = fiddle.make +CLEAN_FILES = +DISTCLEAN_FILES = config.make +MAKING_CLEAN = $(if $(filter %clean,$(MAKECMDGOALS)),1,0) .PHONY: clean distclean clean: -rm -f $(CLEAN_FILES) @@ -53,17 +54,17 @@ distclean: clean ######################################################################## # Special-case builds for which we require certain pre-conditions # which, if not met, may cause warnings or fatal errors in the build. -# This also affects the default optimization level flags. Note that -# the fiddle targets are in this list because they are used for -# generating sqlite.org/fiddle. -OPTIMIZED_TARGETS := dist snapshot fiddle fiddle.debug +# This also affects the default optimization level flags. The fiddle +# targets are in this list because they are used for generating +# sqlite.org/fiddle. +OPTIMIZED_TARGETS = dist snapshot fiddle fiddle.debug ifeq (1,$(MAKING_CLEAN)) - bin.wasm-strip := echo "not stripping" - bin.wasm-opt := irrelevant - bin.emcc := irrelevant - bin.bash := irrelevant - emcc.version := unknown + bin.wasm-strip = echo "not stripping" + bin.wasm-opt = irrelevant + bin.emcc = irrelevant + bin.bash = irrelevant + emcc.version = unknown else # Include config.make and perform some bootstrapping... ifeq (,$(wildcard ./config.make)) @@ -76,7 +77,7 @@ else ifeq (,$(bin.emcc)) $(error Configure script did not find emcc) endif - emcc.version := $(shell $(bin.emcc) --version | sed -n 1p | sed -e 's/^.* \([3-9][^ ]*\) .*$$/\1/;') + emcc.version = $(shell $(bin.emcc) --version | sed -n 1p | sed -e 's/^.* \([3-9][^ ]*\) .*$$/\1/;') $(info using emcc version [$(emcc.version)]) ifeq (,$(bin.wasm-strip)) #################################################################### @@ -98,7 +99,7 @@ else ifneq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) $(error Cannot make release-quality binary because wasm-strip is not available.) endif - bin.wasm-strip := echo "not wasm-stripping" + bin.wasm-strip = echo "not wasm-stripping" endif ifeq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) $(info ==============================================================) @@ -108,7 +109,7 @@ else endif endif # ^^^ end of are-we-MAKING_CLEAN -maybe-wasm-strip := $(bin.wasm-strip) +maybe-wasm-strip = $(bin.wasm-strip) ######################################################################## # JS_BUILD_NAMES exists for documentation purposes only. It enumerates @@ -118,7 +119,7 @@ maybe-wasm-strip := $(bin.wasm-strip) # # - sqlite3-wasmfs = WASMFS-capable library build # -JS_BUILD_NAMES := sqlite3 sqlite3-wasmfs +JS_BUILD_NAMES = sqlite3 sqlite3-wasmfs ######################################################################## # JS_BUILD_MODES exists for documentation purposes only. It enumerates @@ -140,31 +141,31 @@ JS_BUILD_NAMES := sqlite3 sqlite3-wasmfs # that persistent storage (OPFS) is not available in these builds. # These builds are UNTESTED and UNSUPPORTED! # -JS_BUILD_MODES := vanilla esm bunder-friendly node +JS_BUILD_MODES = vanilla esm bunder-friendly node ######################################################################## # dir.top = the top dir of the canonical build tree, where # sqlite3.[ch] live. -dir.top := ../.. +dir.top = ../.. # Maintenance reminder: some Emscripten flags require absolute paths # but we want relative paths for most stuff simply to reduce # noise. The $(abspath...) GNU make function can transform relative # paths to absolute. -dir.wasm := $(patsubst %/,%,$(dir $(MAKEFILE))) -dir.api := api -dir.jacc := jaccwabyt -dir.common := common -dir.fiddle := fiddle -dir.fiddle-debug := fiddle-debug -dir.tool := $(dir.top)/tool +dir.wasm = $(patsubst %/,%,$(dir $(MAKEFILE))) +dir.api = api +dir.jacc = jaccwabyt +dir.common = common +dir.fiddle = fiddle +dir.fiddle-debug = fiddle-debug +dir.tool = $(dir.top)/tool # dir.dout = output dir for deliverables -dir.dout := $(dir.wasm)/jswasm +dir.dout = $(dir.wasm)/jswasm # dir.tmp = output dir for intermediary build files, as opposed to # end-user deliverables. -dir.tmp := $(dir.wasm)/bld -dir.wasmfs := $(dir.dout) +dir.tmp = $(dir.wasm)/bld +dir.wasmfs = $(dir.dout) -MKDIR.bld := $(dir.tmp) +MKDIR.bld = $(dir.tmp) $(MKDIR.bld): @mkdir -p $@ $(dir.dout) @@ -188,17 +189,17 @@ CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ $(dir.fiddle)/*~ # $(sqlite3.canonical.c) must point to the sqlite3.c in # the sqlite3 canonical source tree, as that source file # is required for certain utility and test code. -sqlite3.canonical.c := $(dir.top)/sqlite3.c +sqlite3.canonical.c = $(dir.top)/sqlite3.c sqlite3.c ?= $(firstword $(wildcard $(dir.top)/sqlite3-see.c) $(sqlite3.canonical.c)) -sqlite3.h := $(dir.top)/sqlite3.h +sqlite3.h = $(dir.top)/sqlite3.h ifeq (1,$(MAKING_CLEAN)) - SQLITE_C_IS_SEE := 0 + SQLITE_C_IS_SEE = 0 else ifeq (,$(shell grep sqlite3_activate_see $(sqlite3.c))) - SQLITE_C_IS_SEE := 0 + SQLITE_C_IS_SEE = 0 else - SQLITE_C_IS_SEE := 1 + SQLITE_C_IS_SEE = 1 $(info This is an SEE build) endif endif @@ -216,19 +217,19 @@ $(sqlite3.c): $(sqlite3.h) # barebones=1 disables all "extraneous" stuff from sqlite3-wasm.c, the # goal being to create a WASM file with only the core APIs. ifeq (1,$(barebones)) - wasm-bare-bones := 1 + wasm-bare-bones = 1 $(info ==============================================================) $(info == This is a bare-bones build. It trades away features for) $(info == a smaller .wasm file.) $(info ==============================================================) else - wasm-bare-bones := 0 + wasm-bare-bones = 0 endif # undefine barebones # relatively new gmake feature, not ubiquitous # Common options for building sqlite3-wasm.c and speedtest1.c. # Explicit ENABLEs... -SQLITE_OPT.common := \ +SQLITE_OPT.common = \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_TEMP_STORE=2 \ -DSQLITE_ENABLE_MATH_FUNCTIONS \ @@ -249,7 +250,7 @@ SQLITE_OPT.common := \ SQLITE_OPT.common += -DSQLITE_WASM_ENABLE_C_TESTS # Extra flags for full-featured builds... -SQLITE_OPT.full-featured := \ +SQLITE_OPT.full-featured = \ -DSQLITE_ENABLE_BYTECODE_VTAB \ -DSQLITE_ENABLE_DBPAGE_VTAB \ -DSQLITE_ENABLE_DBSTAT_VTAB \ @@ -260,16 +261,17 @@ SQLITE_OPT.full-featured := \ -DSQLITE_ENABLE_RTREE \ -DSQLITE_ENABLE_SESSION \ -DSQLITE_ENABLE_STMTVTAB \ - -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_COLUMN_METADATA ifeq (0,$(wasm-bare-bones)) # The so-called canonical build is full-featured: - SQLITE_OPT := \ + SQLITE_OPT = \ $(SQLITE_OPT.common) \ $(SQLITE_OPT.full-featured) else # The so-called bare-bones build is exactly that: - SQLITE_OPT := \ + SQLITE_OPT = \ $(SQLITE_OPT.common) \ -DSQLITE_WASM_BARE_BONES # SQLITE_WASM_BARE_BONES tells sqlite3-wasm.c to explicitly omit @@ -344,10 +346,10 @@ endif # See example_extra_init.c for an example implementation. ######################################################################## sqlite3_wasm_extra_init.c ?= $(wildcard sqlite3_wasm_extra_init.c) -cflags.wasm_extra_init := +cflags.wasm_extra_init = ifneq (,$(sqlite3_wasm_extra_init.c)) $(info Enabling SQLITE_EXTRA_INIT via $(sqlite3_wasm_extra_init.c).) - cflags.wasm_extra_init := -DSQLITE_WASM_EXTRA_INIT + cflags.wasm_extra_init = -DSQLITE_WASM_EXTRA_INIT endif ######################################################################### @@ -361,7 +363,7 @@ endif # end result is that the generated JS files may have static version # info from $(bin.version-info) which differ from their runtime-emitted # version info (e.g. from sqlite3_libversion()). -bin.version-info := $(dir.top)/version-info +bin.version-info = $(dir.top)/version-info .NOTPARALLEL: $(bin.version-info) $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile $(MAKE) -C $(dir.top) version-info @@ -372,7 +374,7 @@ $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile # don't need for all builds. That app's -k flag is of particular # importance here, as it allows us to retain the opening comment # block(s), which contain the license header and version info. -bin.stripccomments := $(dir.tool)/stripccomments +bin.stripccomments = $(dir.tool)/stripccomments $(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) $(CC) -o $@ $< DISTCLEAN_FILES += $(bin.stripccomments) @@ -409,7 +411,7 @@ DISTCLEAN_FILES += $(bin.stripccomments) # # -D... flags which should be included in all invocations should be # appended to $(SQLITE.CALL.C-PP.FILTER.global). -bin.c-pp := ./c-pp +bin.c-pp = ./c-pp $(bin.c-pp): c-pp.c $(sqlite3.c) # $(MAKEFILE) $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ @@ -428,13 +430,13 @@ define SQLITE.CALL.C-PP.FILTER $(2): $(1) $$(MAKEFILE_LIST) $$(bin.c-pp) @mkdir -p $$(dir $$@) $$(bin.c-pp) -f $(1) -o $$@ $(3) $(SQLITE.CALL.C-PP.FILTER.global) -#CLEAN_FILES += $(2) +CLEAN_FILES += $(2) endef # /end SQLITE.CALL.C-PP.FILTER ######################################################################## # cflags.common = C compiler flags for all builds -cflags.common := -I. -I$(dir $(sqlite3.c)) +cflags.common = -I. -I$(dir $(sqlite3.c)) # emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API # disables certain features if BigInt is not enabled and such builds # _are not tested_ on any regular basis. @@ -455,9 +457,24 @@ else emcc_opt ?= -Oz endif +# Our JS code installs bindings of each sqlite3_...() WASM export. The +# generated Emscripten JS file does the same using its own framework, +# but we don't use those results and can speed up lib init, and reduce +# memory cost a bit, by stripping them out. Emscripten-side changes +# can "break" this, causing this to be a no-op, but the worst that can +# happen in that case is that it doesn't actually strip anything, +# leading to slightly larger JS files. +# +# This snippet is intended to be used in makefile targets which +# generate an Emscripten module and where $@ is the module's .js/.mjs +# file. +SQLITE.strip-createExportWrapper = \ + sed -i -e '/^.*= \(_sqlite3\|_fiddle\)[^=]* = createExportWrapper/d' $@ || exit; \ + echo '(Probably) stripped out extraneous createExportWrapper() parts.' + # When passing emcc_opt from the CLI, += and re-assignment have no # effect, so emcc_opt+=-g3 doesn't work. So... -emcc_opt_full := $(emcc_opt) -g3 +emcc_opt_full = $(emcc_opt) -g3 # ^^^ ALWAYS use -g3. See below for why. # # ^^^ -flto improves runtime speed at -O0 considerably but doubles @@ -488,31 +505,28 @@ emcc_opt_full := $(emcc_opt) -g3 ######################################################################## # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -EXPORTED_FUNCTIONS.api.core := $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core -EXPORTED_FUNCTIONS.api.in := $(EXPORTED_FUNCTIONS.api.core) +EXPORTED_FUNCTIONS.api.core = $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core +EXPORTED_FUNCTIONS.api.in = $(EXPORTED_FUNCTIONS.api.core) ifeq (1,$(SQLITE_C_IS_SEE)) EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see endif ifeq (0,$(wasm-bare-bones)) EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras endif -EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api +EXPORTED_FUNCTIONS.api = $(dir.tmp)/EXPORTED_FUNCTIONS.api $(EXPORTED_FUNCTIONS.api): $(MKDIR.bld) $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) cat $(EXPORTED_FUNCTIONS.api.in) > $@ ######################################################################## # sqlite3-license-version.js = generated JS file with the license # header and version info. -sqlite3-license-version.js := $(dir.tmp)/sqlite3-license-version.js -# sqlite3-license-version-header.js = JS file containing only the -# license header. -sqlite3-license-version-header.js := $(dir.api)/sqlite3-license-version-header.js +sqlite3-license-version.js = $(dir.tmp)/sqlite3-license-version.js # sqlite3-api-build-version.js = generated JS file which populates the # sqlite3.version object using $(bin.version-info). -sqlite3-api-build-version.js := $(dir.tmp)/sqlite3-api-build-version.js +sqlite3-api-build-version.js = $(dir.tmp)/sqlite3-api-build-version.js # sqlite3-api.jses = the list of JS files which make up # $(sqlite3-api.js.in), in the order they need to be assembled. -sqlite3-api.jses := $(sqlite3-license-version.js) +sqlite3-api.jses = $(sqlite3-license-version.js) # sqlite3-api-prologue.js: initial bootstrapping bits: sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js # whwhasm.js and jaccwabyt.js: Low-level utils, mostly replacing @@ -546,13 +560,13 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # SOAP.js is an external API file which is part of our distribution # but not part of the sqlite3-api.js amalgamation. It's a component of # the first OPFS VFS and necessarily an external file. -SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js -SOAP.js.bld := $(dir.dout)/$(notdir $(SOAP.js)) +SOAP.js = $(dir.api)/sqlite3-opfs-async-proxy.js +SOAP.js.bld = $(dir.dout)/$(notdir $(SOAP.js)) # # $(sqlite3-api.ext.jses) = API-related files which are standalone files, # not part of the amalgamation. # -sqlite3-api.ext.jses := $(SOAP.js.bld) +sqlite3-api.ext.jses = $(SOAP.js.bld) $(SOAP.js.bld): $(SOAP.js) cp $< $@ @@ -572,17 +586,12 @@ $(SOAP.js.bld): $(SOAP.js) # Sidebar: some of the imports are used soley by the Emscripten glue, # which the sqlite3 JS code does not rely on. # -# We build $(sqlite3-api*.*) "because we can" and because it might be -# a useful point of experimentation for some clients, but the -# above-described caveat may well make them unusable for real-life -# clients. -# -# sqlite3-api.js.in = the generated sqlite3-api.js before it gets +# sqlite3-api.js.in = the amalgamated sqlite3-api.js before it gets # preprocessed. It contains all of $(sqlite3-api.jses) but none of the # Emscripten-specific headers and footers. -sqlite3-api.js.in := $(dir.tmp)/sqlite3-api.c-pp.js +sqlite3-api.js.in = $(dir.tmp)/sqlite3-api.c-pp.js $(sqlite3-api.js.in): $(MKDIR.bld) $(sqlite3-api.jses) $(MAKEFILE) - @echo "Making $@..." + @echo "Making $@ ..." @for i in $(sqlite3-api.jses); do \ echo "/* BEGIN FILE: $$i */"; \ cat $$i; \ @@ -591,7 +600,7 @@ $(sqlite3-api.js.in): $(MKDIR.bld) $(sqlite3-api.jses) $(MAKEFILE) ######################################################################## # emcc flags for .c/.o/.wasm/.js. -emcc.flags := +emcc.flags = ifeq (1,$(emcc.verbose)) emcc.flags += -v # -v is _very_ loud but also informative about what it's doing @@ -600,25 +609,25 @@ endif ######################################################################## # emcc flags for .c/.o. -emcc.cflags := +emcc.cflags = emcc.cflags += -std=c99 -fPIC # -------------^^^^^^^^ we need c99 for $(sqlite3-wasm.c), primarily # for variadic macros and snprintf() to implement -# sqlite3_wasm_enum_json(). +# sqlite3__wasm_enum_json(). emcc.cflags += -I. -I$(dir.top) ######################################################################## # emcc flags specific to building .js/.wasm files... -emcc.jsflags := -fPIC +emcc.jsflags = -fPIC emcc.jsflags += --no-entry emcc.jsflags += -sWASM_BIGINT=$(emcc.WASM_BIGINT) emcc.jsflags += -sMODULARIZE emcc.jsflags += -sDYNAMIC_EXECUTION=0 emcc.jsflags += -sNO_POLYFILL emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.api) -emcc.exportedRuntimeMethods := \ - -sEXPORTED_RUNTIME_METHODS=wasmMemory,HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAP64,HEAPU64 +emcc.exportedRuntimeMethods = \ + -sEXPORTED_RUNTIME_METHODS=wasmMemory # wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY -# Emscripten 4.0.7 (2025-04-15) stops exporting HEAP* by default +# Emscripten 4.0.7 (2025-04-15) stops exporting HEAP* by default. emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY @@ -640,10 +649,10 @@ emcc.jsflags += -sSTRICT_JS=0 # tools should be installing, e.g. __syscall_geteuid32 # -sENVIRONMENT values for the various build modes: -emcc.environment.vanilla := web,worker -emcc.environment.bundler-friendly := $(emcc.environment.vanilla) -emcc.environment.esm := $(emcc.environment.vanilla) -emcc.environment.node := node +emcc.environment.vanilla = web,worker +emcc.environment.bundler-friendly = $(emcc.environment.vanilla) +emcc.environment.esm = $(emcc.environment.vanilla) +emcc.environment.node = node # Note that adding ",node" to the list for the other builds causes # Emscripten to generate code which confuses node: it cannot reliably # determine whether the build is for a browser or for node. @@ -665,12 +674,12 @@ emcc.environment.node := node # supported in all configurations (#21071)." # https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md emcc.jsflags += -sALLOW_MEMORY_GROWTH -emcc.INITIAL_MEMORY.128 := 134217728 -emcc.INITIAL_MEMORY.96 := 100663296 -emcc.INITIAL_MEMORY.64 := 67108864 -emcc.INITIAL_MEMORY.32 := 33554432 -emcc.INITIAL_MEMORY.16 := 16777216 -emcc.INITIAL_MEMORY.8 := 8388608 +emcc.INITIAL_MEMORY.128 = 134217728 +emcc.INITIAL_MEMORY.96 = 100663296 +emcc.INITIAL_MEMORY.64 = 67108864 +emcc.INITIAL_MEMORY.32 = 33554432 +emcc.INITIAL_MEMORY.16 = 16777216 +emcc.INITIAL_MEMORY.8 = 8388608 emcc.INITIAL_MEMORY ?= 16 ifeq (,$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY))) $(error emcc.INITIAL_MEMORY must be one of: 8, 16, 32, 64, 96, 128 (megabytes)) @@ -703,7 +712,7 @@ emcc.jsflags += -sSTACK_SIZE=512KB # symbols: we cannot "delete" the Emscripten-defined # $(sqlite3.js.init-func) from vanilla builds (as opposed to ESM # builds) because it's declared with "var". -sqlite3.js.init-func := sqlite3InitModule +sqlite3.js.init-func = sqlite3InitModule emcc.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. #emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() @@ -771,17 +780,17 @@ $(sqlite3-api-build-version.js): $(MKDIR.bld) $(bin.version-info) $(MAKEFILE) # # Maintenance reminder: there are awk binaries out there which do not # support -e SCRIPT. -$(sqlite3-license-version.js): $(MKDIR.bld) $(sqlite3.h) $(sqlite3-license-version-header.js) \ - $(MAKEFILE) +$(sqlite3-license-version.js): $(MKDIR.bld) $(sqlite3.h) \ + $(dir.api)/sqlite3-license-version-header.js $(MAKEFILE) @echo "Making $@..."; { \ - cat $(sqlite3-license-version-header.js); \ + cat $(dir.api)/sqlite3-license-version-header.js; \ echo '/*'; \ echo '** This code was built from sqlite3 version...'; \ echo "**"; \ awk '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' $(sqlite3.h); \ awk '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ echo "**"; \ - echo "** Using the Emscripten SDK version $(emcc.version)."; \ + echo "** with the help of Emscripten SDK version $(emcc.version)."; \ echo '*/'; \ } > $@ @@ -793,9 +802,9 @@ $(sqlite3-license-version.js): $(MKDIR.bld) $(sqlite3.h) $(sqlite3-license-versi # --post-js injects code which runs after the WASM module is loaded # and includes the entirety of the library plus some # Emscripten-specific post-bootstrapping code. -pre-js.js.in := $(dir.api)/pre-js.c-pp.js -post-js.js.in := $(dir.tmp)/post-js.c-pp.js -post-jses.js := \ +pre-js.js.in = $(dir.api)/pre-js.c-pp.js +post-js.js.in = $(dir.tmp)/post-js.c-pp.js +post-jses.js = \ $(dir.api)/post-js-header.js \ $(sqlite3-api.js.in) \ $(dir.api)/post-js-footer.js @@ -811,10 +820,10 @@ $(post-js.js.in): $(MKDIR.bld) $(post-jses.js) $(MAKEFILE) # Undocumented Emscripten feature: if the target file extension is # "mjs", it defaults to ES6 module builds: # https://github.com/emscripten-core/emscripten/issues/14383 -sqlite3.wasm := $(dir.dout)/sqlite3.wasm -sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c -sqlite3-wasm.cfiles := $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) -sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles) +sqlite3.wasm = $(dir.dout)/sqlite3.wasm +sqlite3-wasm.c = $(dir.api)/sqlite3-wasm.c +sqlite3-wasm.cfiles = $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) +sqlite3-wasmfs.cfiles = $(sqlite3-wasm.cfiles) # sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter # (predictably) results in a slightly faster binary. We're close # enough to the target speed requirements that the 500ms makes a @@ -859,17 +868,9 @@ if [ x1 = x$(1) ]; then \ fi endef -sqlite3-api.js := $(dir.dout)/sqlite3-api.js -sqlite3.js := $(dir.dout)/sqlite3.js -sqlite3-api.mjs := $(dir.dout)/sqlite3-api.mjs -sqlite3.mjs := $(dir.dout)/sqlite3.mjs -sqlite3-api-bundler-friendly.mjs := $(dir.dout)/sqlite3-api-bundler-friendly.mjs -sqlite3-bundler-friendly.mjs := $(dir.dout)/sqlite3-bundler-friendly.mjs -sqlite3-api-node.mjs := $(dir.dout)/sqlite3-api-node.mjs -sqlite3-node.mjs := $(dir.dout)/sqlite3-node.mjs -sqlite3-api-wasmfs.mjs := $(dir.tmp)/sqlite3-api-wasmfs.mjs -sqlite3-wasmfs.mjs := $(dir.wasmfs)/sqlite3-wasmfs.mjs -EXPORTED_FUNCTIONS.fiddle := $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle +sqlite3.js = $(dir.dout)/sqlite3.js +sqlite3.mjs = $(dir.dout)/sqlite3.mjs +EXPORTED_FUNCTIONS.fiddle = $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle # The various -D... values used by *.c-pp.js include: # @@ -903,24 +904,10 @@ EXPORTED_FUNCTIONS.fiddle := $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle # build time). $(sqlite3.wasm): $(sqlite3.js) $(sqlite3.mjs): $(sqlite3.js) -$(sqlite3-bundler-friendly.mjs): $(sqlite3.mjs) -$(sqlite3-node.mjs): $(sqlite3.mjs) +$(dir.dout)/sqlite3-bundler-friendly.mjs: $(sqlite3.mjs) +$(dir.dout)/sqlite3-node.mjs: $(sqlite3.mjs) #CLEAN_FILES += $(sqlite3.wasm) -######################################################################## -# We need separate copies of certain supplementary JS files for the -# bundler-friendly build. Concretely, any supplemental JS files which -# themselves use importScripts() or Workers or URL() constructors -# which refer to other in-tree (m)JS files require a bundler-friendly -# copy. -sqlite3-worker1.js.in := $(dir.api)/sqlite3-worker1.c-pp.js -sqlite3-worker1-promiser.js.in := $(dir.api)/sqlite3-worker1-promiser.c-pp.js -sqlite3-worker1.js := $(dir.dout)/sqlite3-worker1.js -sqlite3-worker1-promiser.js := $(dir.dout)/sqlite3-worker1-promiser.js -sqlite3-worker1-promiser.mjs := $(dir.dout)/sqlite3-worker1-promiser.mjs -sqlite3-worker1-bundler-friendly.mjs := $(dir.dout)/sqlite3-worker1-bundler-friendly.mjs -sqlite3-worker1-promiser-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-promiser-bundler-friendly.js - ifneq (1,$(MAKING_CLEAN)) # This block MUST come between the above definitions of # sqlite3-...js/mjs and the $(eval) calls below this block which use @@ -933,7 +920,10 @@ ifneq (1,$(MAKING_CLEAN)) # the $ references in those languages made it just as illegible as the # native makefile code. Somewhat surprisingly, moving that code generation # to C makes it slightly less illegible than the previous 3 options. -bin.mkwb := ./mkwasmbuilds +# +# Maintenance note: the various $(c-pp.D.XYZ) vars are defined in this +# step. +bin.mkwb = ./mkwasmbuilds $(bin.mkwb): $(bin.mkwb).c $(MAKEFILE) $(CC) -o $@ $< DISTCLEAN_FILES += $(bin.mkwb) @@ -945,45 +935,60 @@ DISTCLEAN_FILES += $(bin.mkwb) endif DISTCLEAN_FILES += .wasmbuilds.make -$(eval $(call SQLITE.CALL.C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1.js))) -$(eval $(call SQLITE.CALL.C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1-bundler-friendly.mjs),\ +######################################################################## +# We need separate copies of certain supplementary JS files for the +# bundler-friendly build. Concretely, any supplemental JS files which +# themselves use importScripts() or Workers or URL() constructors +# which refer to other in-tree (m)JS files require a bundler-friendly +# copy. Bundler-friendly builds replace certain references to string +# vars/expressions with string literals, as bundler tools are static +# code analyzers and cannot cope with the former. +# +# Most of what follows is the generation of those copies. +$(eval $(call SQLITE.CALL.C-PP.FILTER,$(dir.api)/sqlite3-worker1.c-pp.js,\ + $(dir.dout)/sqlite3-worker1.js)) +$(eval $(call SQLITE.CALL.C-PP.FILTER,$(dir.api)/sqlite3-worker1.c-pp.js,\ + $(dir.dout)/sqlite3-worker1-bundler-friendly.mjs,\ $(c-pp.D.sqlite3-bundler-friendly))) -$(eval $(call SQLITE.CALL.C-PP.FILTER,$(sqlite3-worker1-promiser.js.in),$(sqlite3-worker1-promiser.js))) -$(eval $(call SQLITE.CALL.C-PP.FILTER,$(sqlite3-worker1-promiser.js.in),\ - $(sqlite3-worker1-promiser-bundler-friendly.js),\ +$(eval $(call SQLITE.CALL.C-PP.FILTER,$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ + $(dir.dout)/sqlite3-worker1-promiser.js)) +$(eval $(call SQLITE.CALL.C-PP.FILTER,$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ + $(dir.dout)/sqlite3-worker1-promiser-bundler-friendly.js,\ $(c-pp.D.sqlite3-bundler-friendly))) -$(eval $(call SQLITE.CALL.C-PP.FILTER,$(sqlite3-worker1-promiser.js.in),$(sqlite3-worker1-promiser.mjs),\ +$(eval $(call SQLITE.CALL.C-PP.FILTER,$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ + $(dir.dout)/sqlite3-worker1-promiser.mjs,\ -Dtarget=es6-module -Dtarget=es6-bundler-friendly)) -$(sqlite3-bundler-friendly.mjs): $(sqlite3-worker1-bundler-friendly.mjs) \ - $(sqlite3-worker1-promiser-bundler-friendly.js) $(eval $(call SQLITE.CALL.C-PP.FILTER,demo-worker1-promiser.c-pp.js,demo-worker1-promiser.js)) $(eval $(call SQLITE.CALL.C-PP.FILTER,demo-worker1-promiser.c-pp.js,demo-worker1-promiser.mjs,\ -Dtarget=es6-module)) $(eval $(call SQLITE.CALL.C-PP.FILTER,demo-worker1-promiser.c-pp.html,demo-worker1-promiser.html)) $(eval $(call SQLITE.CALL.C-PP.FILTER,demo-worker1-promiser.c-pp.html,demo-worker1-promiser-esm.html,\ -Dtarget=es6-module)) -all: $(sqlite3-worker1.js) \ - $(sqlite3-worker1-promiser.js) $(sqlite3-worker1-promiser.mjs) -demo-worker1-promiser.html: $(sqlite3-worker1-promiser.js) demo-worker1-promiser.js +$(dir.dout)/sqlite3-bundler-friendly.mjs: \ + $(dir.dout)/sqlite3-worker1-bundler-friendly.mjs \ + $(dir.dout)/sqlite3-worker1-promiser-bundler-friendly.js + +demo-worker1-promiser.html: $(dir.dout)/sqlite3-worker1-promiser.js demo-worker1-promiser.js demo-worker1-promiser-esm.html: $(sqlite3-worker1-promiser.mjs) demo-worker1-promiser.mjs all: demo-worker1-promiser.html demo-worker1-promiser-esm.html sqlite3-api.ext.jses += \ - $(sqlite3-worker1-promiser.mjs) \ - $(sqlite3-worker1-bundler-friendly.mjs) \ - $(sqlite3-worker1.js) + $(dir.dout)/sqlite3-worker1-promiser.mjs \ + $(dir.dout)/sqlite3-worker1-promiser.js \ + $(dir.dout)/sqlite3-worker1-bundler-friendly.mjs \ + $(dir.dout)/sqlite3-worker1.js all quick: $(sqlite3-api.ext.jses) q: quick ######################################################################## # batch-runner.js is part of one of the test apps which reads in SQL # dumps generated by $(speedtest1) and executes them. -dir.sql := sql -speedtest1 := ../../speedtest1 -speedtest1.c := ../../test/speedtest1.c -speedtest1.sql := $(dir.sql)/speedtest1.sql -speedtest1.cliflags := --size 10 --big-transactions +dir.sql = sql +speedtest1 = ../../speedtest1 +speedtest1.c = ../../test/speedtest1.c +speedtest1.sql = $(dir.sql)/speedtest1.sql +speedtest1.cliflags = --size 10 --big-transactions $(speedtest1): $(MAKE) -C ../.. speedtest1 $(speedtest1.sql): $(speedtest1) $(MAKEFILE) @@ -1004,8 +1009,8 @@ batch: batch-runner.list # # emcc.speedtest1.common = emcc flags used by multiple builds of speedtest1 # emcc.speedtest1 = emcc flags used by main build of speedtest1 -emcc.speedtest1.common := $(emcc_opt_full) -emcc.speedtest1 := -I. -I$(dir $(sqlite3.canonical.c)) +emcc.speedtest1.common = $(emcc_opt_full) +emcc.speedtest1 = -I. -I$(dir $(sqlite3.canonical.c)) emcc.speedtest1 += -sENVIRONMENT=web emcc.speedtest1 += -sALLOW_MEMORY_GROWTH emcc.speedtest1 += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) @@ -1018,7 +1023,7 @@ emcc.speedtest1.common += -Wno-limited-postlink-optimizations emcc.speedtest1.common += -Wno-unused-main # ^^^^ -Wno-unused-main is for emcc 3.1.52+. speedtest1 has a wasm_main() which is # exported and called by the JS code. -EXPORTED_FUNCTIONS.speedtest1 := $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) +EXPORTED_FUNCTIONS.speedtest1 = $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) emcc.speedtest1.common += -sSTACK_SIZE=512KB emcc.speedtest1.common += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.speedtest1) emcc.speedtest1.common += $(emcc.exportedRuntimeMethods) @@ -1027,8 +1032,8 @@ emcc.speedtest1.common += -sDYNAMIC_EXECUTION=0 emcc.speedtest1.common += --minify 0 emcc.speedtest1.common += -sEXPORT_NAME=$(sqlite3.js.init-func) emcc.speedtest1.common += -sWASM_BIGINT=$(emcc.WASM_BIGINT) -speedtest1.exit-runtime0 := -sEXIT_RUNTIME=0 -speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 +speedtest1.exit-runtime0 = -sEXIT_RUNTIME=0 +speedtest1.exit-runtime1 = -sEXIT_RUNTIME=1 # Re -sEXIT_RUNTIME=1 vs 0: if it's 1 and speedtest1 crashes, we get # this error from emscripten: # @@ -1049,10 +1054,9 @@ speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 $(EXPORTED_FUNCTIONS.speedtest1): $(MKDIR.bld) $(EXPORTED_FUNCTIONS.api.core) @echo "Making $@ ..." @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.core); } > $@ -speedtest1.js := $(dir.dout)/speedtest1.js -speedtest1.wasm := $(dir.dout)/speedtest1.wasm -emcc.flags.speedtest1-vanilla := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM -speedtest1.cfiles := $(speedtest1.c) $(sqlite3-wasm.c) +speedtest1.js = $(dir.dout)/speedtest1.js +emcc.flags.speedtest1-vanilla = $(cflags.common) -DSQLITE_SPEEDTEST1_WASM +speedtest1.cfiles = $(speedtest1.c) $(sqlite3-wasm.c) $(speedtest1.js): $(MAKEFILE) $(speedtest1.cfiles) \ $(pre-post-speedtest1-vanilla.deps) \ $(EXPORTED_FUNCTIONS.speedtest1) @@ -1066,14 +1070,13 @@ $(speedtest1.js): $(MAKEFILE) $(speedtest1.cfiles) \ -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cfiles) -lm - $(maybe-wasm-strip) $(speedtest1.wasm) - sed -i -e '/^var _sqlite3.*createExportWrapper/d' $@ - chmod -x $(speedtest1.wasm) - ls -la $@ $(speedtest1.wasm) + @chmod -x $(basename $@).wasm + @$(maybe-wasm-strip) $(basename $@).wasm + @$(SQLITE.strip-createExportWrapper) + @ls -la $@ $(speedtest1.wasm) speedtest1: $(speedtest1.js) all: speedtest1 -#CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) # end speedtest1.js ######################################################################## @@ -1098,7 +1101,7 @@ $(eval $(call SQLITE.CALL.C-PP.FILTER,tester1.c-pp.js,tester1.mjs,$(c-pp.D.sqlit $(eval $(call SQLITE.CALL.C-PP.FILTER,tester1.c-pp.html,tester1.html)) $(eval $(call SQLITE.CALL.C-PP.FILTER,tester1.c-pp.html,tester1-esm.html,$(c-pp.D.sqlite3-esm))) tester1: tester1.js tester1.mjs tester1.html tester1-esm.html -# Note that we do not include $(sqlite3-bundler-friendly.mjs) in this +# Note that we do not include $(dir.dout)/sqlite3-bundler-friendly.mjs in this # because bundlers are client-specific. all quick: tester1 quick: $(sqlite3.js) @@ -1110,7 +1113,7 @@ quick: $(sqlite3.js) # painful. .PHONY: o0 o1 o2 o3 os oz -emcc-opt-extra := +emcc-opt-extra = #ifeq (1,$(wasm-bare-bones)) #emcc-opt-extra += -flto # ^^^^ -flto can have a considerably performance boost at -O0 but @@ -1138,7 +1141,7 @@ oz: clean # Sub-makes... # sqlite.org/fiddle application... -include fiddle.make +include $(MAKEFILE.fiddle) # Only add wasmfs if wasmfs.enable=1 or we're running (dist)clean ifneq (,$(filter wasmfs,$(MAKECMDGOALS))) @@ -1156,16 +1159,17 @@ ifeq (1,$(wasmfs.enable)) # little benefit. # ######################################################################## -# Some platforms do not support the WASMFS build. Raspberry Pi OS is one -# of them. As such platforms are discovered, add their (uname -m) name -# to PLATFORMS_WITH_NO_WASMFS to exclude the wasmfs build parts. -PLATFORMS_WITH_NO_WASMFS := aarch64 # add any others here -THIS_ARCH := $(shell /usr/bin/uname -m) +# Some platforms do not support the WASMFS build. Raspberry Pi OS is +# one of them (or was when that comment was initially written). As +# such platforms are discovered, add their (uname -m) name to +# PLATFORMS_WITH_NO_WASMFS to exclude the wasmfs build parts. +PLATFORMS_WITH_NO_WASMFS = aarch64 # add any others here +THIS_ARCH = $(shell /usr/bin/uname -m) ifneq (,$(filter $(THIS_ARCH),$(PLATFORMS_WITH_NO_WASMFS))) $(info This platform does not support the WASMFS build.) -HAVE_WASMFS := 0 +HAVE_WASMFS = 0 else -HAVE_WASMFS := 1 +HAVE_WASMFS = 1 include wasmfs.make endif endif @@ -1203,7 +1207,7 @@ update-docs: echo "Pass wasm.docs.home=/path/to/wasm/docs/checkout or edit this makefile to suit."; \ exit 127 else -wasm.docs.jswasm := $(wasm.docs.home)/jswasm +wasm.docs.jswasm = $(wasm.docs.home)/jswasm update-docs: $(bin.stripccomments) $(sqlite3.js) $(sqlite3.wasm) @echo "Copying files to the /wasm docs. Be sure to use an -Oz build for this!" cp $(sqlite3.wasm) $(wasm.docs.jswasm)/. diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras index e635d93b3..01dad072e 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras @@ -1,12 +1,27 @@ +_sqlite3_column_database_name +_sqlite3_column_origin_name +_sqlite3_column_table_name +_sqlite3_create_module +_sqlite3_create_module_v2 _sqlite3_create_window_function -_sqlite3_progress_handler -_sqlite3_set_authorizer +_sqlite3_declare_vtab +_sqlite3_drop_modules _sqlite3_preupdate_blobwrite _sqlite3_preupdate_count _sqlite3_preupdate_depth _sqlite3_preupdate_hook _sqlite3_preupdate_new _sqlite3_preupdate_old +_sqlite3_progress_handler +_sqlite3_set_authorizer +_sqlite3_vtab_collation +_sqlite3_vtab_distinct +_sqlite3_vtab_in +_sqlite3_vtab_in_first +_sqlite3_vtab_in_next +_sqlite3_vtab_nochange +_sqlite3_vtab_on_conflict +_sqlite3_vtab_rhs_value _sqlite3changegroup_add _sqlite3changegroup_add_strm _sqlite3changegroup_delete @@ -49,15 +64,3 @@ _sqlite3session_object_config _sqlite3session_patchset _sqlite3session_patchset_strm _sqlite3session_table_filter -_sqlite3_create_module -_sqlite3_create_module_v2 -_sqlite3_declare_vtab -_sqlite3_drop_modules -_sqlite3_vtab_collation -_sqlite3_vtab_distinct -_sqlite3_vtab_in -_sqlite3_vtab_in_first -_sqlite3_vtab_in_next -_sqlite3_vtab_nochange -_sqlite3_vtab_on_conflict -_sqlite3_vtab_rhs_value diff --git a/ext/wasm/api/sqlite3-api-glue.c-pp.js b/ext/wasm/api/sqlite3-api-glue.c-pp.js index a38b9cb5e..8d2d4a589 100644 --- a/ext/wasm/api/sqlite3-api-glue.c-pp.js +++ b/ext/wasm/api/sqlite3-api-glue.c-pp.js @@ -20,7 +20,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; - const toss3 = sqlite3.SQLite3Error.toss; const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; globalThis.WhWasmUtilInstaller(wasm); delete globalThis.WhWasmUtilInstaller; @@ -368,6 +367,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ); }/* sqlite3_set_authorizer() */ + if( !!wasm.exports.sqlite3_column_origin_name ){ + wasm.bindingSignatures.push( + ["sqlite3_column_database_name","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_origin_name","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_table_name","string", "sqlite3_stmt*", "int"] + ); + } + if(false && wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ /* ^^^ "the problem" is that this is an optional feature and the build-time function-export list does not currently take diff --git a/ext/wasm/api/sqlite3-api-oo1.c-pp.js b/ext/wasm/api/sqlite3-api-oo1.c-pp.js index 3d6a24c77..8663dcdde 100644 --- a/ext/wasm/api/sqlite3-api-oo1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-oo1.c-pp.js @@ -13,10 +13,9 @@ This file contains the so-called OO #1 API wrapper for the sqlite3 WASM build. It requires that sqlite3-api-glue.js has already run - and it installs its deliverable as globalThis.sqlite3.oo1. + and it installs its deliverable as sqlite3.oo1. */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - const toss = (...args)=>{throw new Error(args.join(' '))}; const toss3 = (...args)=>{throw new sqlite3.SQLite3Error(...args)}; const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; @@ -39,6 +38,21 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ const __ptrMap = new WeakMap(); /** + A Set of oo1.DB or oo1.Stmt objects which are proxies for + (sqlite3*) resp. (sqlite3_stmt*) pointers which themselves are + owned elsewhere. Objects in this Set do not own their underlying + handle and that handle must be guaranteed (by the client) to + outlive the proxy. DB.close()/Stmt.finalize() methods will remove + the object from this Set _instead_ of closing/finalizing the + pointer. These proxies are primarily intended as a way to briefly + wrap an (sqlite3[_stmt]*) object as an oo1.DB/Stmt without taking + over ownership, to take advantage of simplifies usage compared to + the C API while not imposing any change of ownership. + + See DB.wrapHandle() and Stmt.wrapHandle(). + */ + const __doesNotOwnHandle = new Set(); + /** Map of DB instances to objects, each object being a map of Stmt wasm pointers to Stmt objects. */ @@ -235,73 +249,89 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; } const opt = ctor.normalizeArgs(...args); - let fn = opt.filename, vfsName = opt.vfs, flagsStr = opt.flags; - if(('string'!==typeof fn && 'number'!==typeof fn) - || 'string'!==typeof flagsStr - || (vfsName && ('string'!==typeof vfsName && 'number'!==typeof vfsName))){ - sqlite3.config.error("Invalid DB ctor args",opt,arguments); - toss3("Invalid arguments for DB constructor."); - } - let fnJs = ('number'===typeof fn) ? wasm.cstrToJs(fn) : fn; - const vfsCheck = ctor._name2vfs[fnJs]; - if(vfsCheck){ - vfsName = vfsCheck.vfs; - fn = fnJs = vfsCheck.filename(fnJs); - } - let pDb, oflags = 0; - if( flagsStr.indexOf('c')>=0 ){ - oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; - } - if( flagsStr.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; - if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; - oflags |= capi.SQLITE_OPEN_EXRESCODE; - const stack = wasm.pstack.pointer; - try { - const pPtr = wasm.pstack.allocPtr() /* output (sqlite3**) arg */; - let rc = capi.sqlite3_open_v2(fn, pPtr, oflags, vfsName || 0); - pDb = wasm.peekPtr(pPtr); - checkSqlite3Rc(pDb, rc); - capi.sqlite3_extended_result_codes(pDb, 1); - if(flagsStr.indexOf('t')>=0){ - capi.sqlite3_trace_v2(pDb, capi.SQLITE_TRACE_STMT, - __dbTraceToConsole, pDb); + //sqlite3.config.debug("DB ctor",opt); + let pDb; + if( (pDb = opt['sqlite3*']) ){ + /* This property ^^^^^ is very specifically NOT DOCUMENTED and + NOT part of the public API. This is a back door for functions + like DB.wrapDbHandle(). */ + //sqlite3.config.debug("creating proxy db from",opt); + if( !opt['sqlite3*:takeOwnership'] ){ + /* This is object does not own its handle. */ + __doesNotOwnHandle.add(this); } - }catch( e ){ - if( pDb ) capi.sqlite3_close_v2(pDb); - throw e; - }finally{ - wasm.pstack.restore(stack); + this.filename = capi.sqlite3_db_filename(pDb,'main'); + }else{ + let fn = opt.filename, vfsName = opt.vfs, flagsStr = opt.flags; + if(('string'!==typeof fn && 'number'!==typeof fn) + || 'string'!==typeof flagsStr + || (vfsName && ('string'!==typeof vfsName && 'number'!==typeof vfsName))){ + sqlite3.config.error("Invalid DB ctor args",opt,arguments); + toss3("Invalid arguments for DB constructor."); + } + let fnJs = ('number'===typeof fn) ? wasm.cstrToJs(fn) : fn; + const vfsCheck = ctor._name2vfs[fnJs]; + if(vfsCheck){ + vfsName = vfsCheck.vfs; + fn = fnJs = vfsCheck.filename(fnJs); + } + let oflags = 0; + if( flagsStr.indexOf('c')>=0 ){ + oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + } + if( flagsStr.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; + if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; + oflags |= capi.SQLITE_OPEN_EXRESCODE; + const stack = wasm.pstack.pointer; + try { + const pPtr = wasm.pstack.allocPtr() /* output (sqlite3**) arg */; + let rc = capi.sqlite3_open_v2(fn, pPtr, oflags, vfsName || 0); + pDb = wasm.peekPtr(pPtr); + checkSqlite3Rc(pDb, rc); + capi.sqlite3_extended_result_codes(pDb, 1); + if(flagsStr.indexOf('t')>=0){ + capi.sqlite3_trace_v2(pDb, capi.SQLITE_TRACE_STMT, + __dbTraceToConsole, pDb); + } + }catch( e ){ + if( pDb ) capi.sqlite3_close_v2(pDb); + throw e; + }finally{ + wasm.pstack.restore(stack); + } + this.filename = fnJs; } - this.filename = fnJs; __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); - try{ + if( !opt['sqlite3*'] ){ + try{ //#if enable-see - dbCtorApplySEEKey(this,opt); + dbCtorApplySEEKey(this,opt); //#endif - // Check for per-VFS post-open SQL/callback... - const pVfs = capi.sqlite3_js_db_vfs(pDb) - || toss3("Internal error: cannot get VFS for new db handle."); - const postInitSql = __vfsPostOpenCallback[pVfs]; - if(postInitSql){ - /** - Reminder: if this db is encrypted and the client did _not_ pass - in the key, any init code will fail, causing the ctor to throw. - We don't actually know whether the db is encrypted, so we cannot - sensibly apply any heuristics which skip the init code only for - encrypted databases for which no key has yet been supplied. - */ - if(postInitSql instanceof Function){ - postInitSql(this, sqlite3); - }else{ - checkSqlite3Rc( - pDb, capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0) - ); + // Check for per-VFS post-open SQL/callback... + const pVfs = capi.sqlite3_js_db_vfs(pDb) + || toss3("Internal error: cannot get VFS for new db handle."); + const postInitSql = __vfsPostOpenCallback[pVfs]; + if(postInitSql){ + /** + Reminder: if this db is encrypted and the client did _not_ pass + in the key, any init code will fail, causing the ctor to throw. + We don't actually know whether the db is encrypted, so we cannot + sensibly apply any heuristics which skip the init code only for + encrypted databases for which no key has yet been supplied. + */ + if(postInitSql instanceof Function){ + postInitSql(this, sqlite3); + }else{ + checkSqlite3Rc( + pDb, capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0) + ); + } } + }catch(e){ + this.close(); + throw e; } - }catch(e){ - this.close(); - throw e; } }; @@ -404,7 +434,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - `vfs`: the VFS fname //#if enable-see - SEE-capable builds optionally support ONE of the following additional options: @@ -430,7 +459,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ is supplied and the database is encrypted, execution of the post-initialization SQL will fail, causing the constructor to throw. - //#endif enable-see The `filename` and `vfs` arguments may be either JS strings or @@ -458,8 +486,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Internal-use enum for mapping JS types to DB-bindable types. These do not (and need not) line up with the SQLITE_type - values. All values in this enum must be truthy and distinct - but they need not be numbers. + values. All values in this enum must be truthy and (mostly) + distinct but they need not be numbers. */ const BindTypes = { null: 1, @@ -468,7 +496,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ boolean: 4, blob: 5 }; - BindTypes['undefined'] == BindTypes.null; if(wasm.bigIntEnabled){ BindTypes.bigint = BindTypes.number; } @@ -487,26 +514,30 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - `db`: the DB object which created the statement. - `columnCount`: the number of result columns in the query, or 0 - for queries which cannot return results. This property is a proxy - for sqlite3_column_count() and its use in loops should be avoided - because of the call overhead associated with that. The - `columnCount` is not cached when the Stmt is created because a - schema change made via a separate db connection between this - statement's preparation and when it is stepped may invalidate it. + for queries which cannot return results. This property is a + read-only proxy for sqlite3_column_count() and its use in loops + should be avoided because of the call overhead associated with + that. The `columnCount` is not cached when the Stmt is created + because a schema change made between this statement's preparation + and when it is stepped may invalidate it. - - `parameterCount`: the number of bindable parameters in the query. + - `parameterCount`: the number of bindable parameters in the + query. Like `columnCount`, this property is ready-only and is a + proxy for a C API call. As a general rule, most methods of this class will throw if called on an instance which has been finalized. For brevity's sake, the method docs do not all repeat this warning. */ - const Stmt = function(){ + const Stmt = function(/*oo1db, stmtPtr, BindTypes [,takeOwnership=true] */){ if(BindTypes!==arguments[2]){ toss3(capi.SQLITE_MISUSE, "Do not call the Stmt constructor directly. Use DB.prepare()."); } this.db = arguments[0]; __ptrMap.set(this, arguments[1]); - this.parameterCount = capi.sqlite3_bind_parameter_count(this.pointer); + if( arguments.length>3 && !arguments[3] ){ + __doesNotOwnHandle.add(this); + } }; /** Throws if the given DB has been closed, else it is returned. */ @@ -699,10 +730,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }, /** Finalizes all open statements and closes this database - connection. This is a no-op if the db has already been - closed. After calling close(), `this.pointer` will resolve to - `undefined`, so that can be used to check whether the db - instance is still opened. + connection (with one exception noted below). This is a no-op if + the db has already been closed. After calling close(), + `this.pointer` will resolve to `undefined`, and that can be + used to check whether the db instance is still opened. If this.onclose.before is a function then it is called before any close-related cleanup. @@ -722,14 +753,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ all, will never trigger close(), so onclose handlers are not a reliable way to implement close-time cleanup or maintenance of a db. + + If this instance was created using DB.wrapHandle() and does not + own this.pointer then it does not close the db handle but it + does perform all other work, such as calling onclose callbacks + and disassociating this object from this.pointer. */ close: function(){ - if(this.pointer){ + const pDb = this.pointer; + if(pDb){ if(this.onclose && (this.onclose.before instanceof Function)){ try{this.onclose.before(this)} catch(e){/*ignore*/} } - const pDb = this.pointer; Object.keys(__stmtMap.get(this)).forEach((k,s)=>{ if(s && s.pointer){ try{s.finalize()} @@ -738,7 +774,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }); __ptrMap.delete(this); __stmtMap.delete(this); - capi.sqlite3_close_v2(pDb); + if( !__doesNotOwnHandle.delete(this) ){ + capi.sqlite3_close_v2(pDb); + } if(this.onclose && (this.onclose.after instanceof Function)){ try{this.onclose.after(this)} catch(e){/*ignore*/} @@ -1061,18 +1099,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const cbArgCache = Object.create(null) /* 2nd arg for arg.cbArg, used by (at least) row-to-object converter */; - for(; stmt.step(); stmt._lockedByExec = false){ + for( ; stmt.step(); __execLock.delete(stmt) ){ if(0===gotColNames++){ stmt.getColumnNames(cbArgCache.columnNames = (opt.columnNames || [])); } - stmt._lockedByExec = true; + __execLock.add(stmt); const row = arg.cbArg(stmt,cbArgCache); if(resultRows) resultRows.push(row); if(callback && false === callback.call(opt, row, stmt)){ break; } } - stmt._lockedByExec = false; + __execLock.delete(stmt); } if(0===gotColNames){ /* opt.columnNames was provided but we visited no result rows */ @@ -1094,7 +1132,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }*/finally{ wasm.scopedAllocPop(stack); if(stmt){ - delete stmt._lockedByExec; + __execLock.delete(stmt); stmt.finalize(); } } @@ -1388,7 +1426,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Starts a transaction, calls the given callback, and then either - rolls back or commits the savepoint, depending on whether the + rolls back or commits the transaction, depending on whether the callback throws. The callback is passed this db object as its only argument. On success, returns the result of the callback. Throws on error. @@ -1451,9 +1489,63 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ checkRc: function(resultCode){ return checkSqlite3Rc(this, resultCode); - } + }, }/*DB.prototype*/; + /** + Returns a new oo1.DB instance which wraps the given (sqlite3*) + WASM pointer, optionally with or without taking over ownership of + that pointer. + + The first argument must be either a non-NULL (sqlite3*) WASM + pointer. + + The second argument, defaulting to false, specifies ownership of + the first argument. If it is truthy, the returned object will + pass that pointer to sqlite3_close() when its close() method is + called, otherwise it will not. + + Throws if pDb is not a non-0 WASM pointer. + + The caller MUST GUARANTEE that the passed-in handle will outlive + the returned object, i.e. that it will not be closed. If it is closed, + this object will hold a stale pointer and results are undefined. + + Aside from its lifetime, the proxy is to be treated as any other + DB instance, including the requirement of calling close() on + it. close() will free up internal resources owned by the proxy + and disassociate the proxy from that handle but will not + actually close the proxied db handle unless this function is + passed a thruthy second argument. + + To stress: + + - DO NOT call sqlite3_close() (or similar) on the being-proxied + pointer while a proxy is active. + + - ALWAYS eventually call close() on the returned object. If the + proxy does not own the underlying handle then its MUST be + closed BEFORE the being-proxied handle is closed. + + Design notes: + + - wrapHandle() "could" accept a DB object instance as its first + argument and proxy thatDb.pointer but there is currently no use + case where doing so would be useful, so it does not allow + that. That restriction may be lifted in a future version. + */ + DB.wrapHandle = function(pDb, takeOwnership=false){ + if( !pDb || !wasm.isPtr(pDb) ){ + throw new sqlite3.SQLite3Error(capi.SQLITE_MISUSE, + "Argument must be a WASM sqlite3 pointer"); + } + return new DB({ + /* This ctor call style is very specifically internal-use-only. + It is not documented and may change at any time. */ + "sqlite3*": pDb, + "sqlite3*:takeOwnership": !!takeOwnership + }); + }; /** Throws if the given Stmt has been finalized, else stmt is returned. */ @@ -1475,8 +1567,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ case BindTypes.string: return t; case BindTypes.bigint: - if(wasm.bigIntEnabled) return t; - /* else fall through */ + return wasm.bigIntEnabled ? t : undefined; default: return util.isBindableTypedArray(v) ? BindTypes.blob : undefined; } @@ -1511,7 +1602,30 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; /** - If stmt._lockedByExec is truthy, this throws an exception + Each Stmt object which is "locked" by DB.exec() gets an entry + here to note that "lock". + + The reason this is in place is because exec({callback:...})'s + callback gets access to the Stmt objects created internally by + exec() but it must not use certain Stmt APIs. + */ + const __execLock = new Set(); + /** + This is a Stmt.get() counterpart of __execLock. Each time + Stmt.step() returns true, the statement is added to this set, + indicating that Stmt.get() is legal. Stmt APIs which invalidate + that status remove the Stmt object from this set, which will + cause Stmt.get() to throw with a descriptive error message + instead of a more generic "API misuse" if we were to allow that + call to reach the C API. + */ + const __stmtMayGet = new Set(); + + /** + Stmt APIs which are prohibited on locked objects must call + affirmNotLockedByExec() before doing any work. + + If __execLock.has(stmt) is truthy, this throws an exception complaining that the 2nd argument (an operation name, e.g. "bind()") is not legal while the statement is "locked". Locking happens before an exec()-like callback is passed a @@ -1519,7 +1633,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ finalize the statement. If it does not throw, it returns stmt. */ const affirmNotLockedByExec = function(stmt,currentOpName){ - if(stmt._lockedByExec){ + if(__execLock.has(stmt)){ toss3("Operation is illegal when statement is locked:",currentOpName); } return stmt; @@ -1604,7 +1718,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ toss3("Unsupported bind() argument type: "+(typeof val)); } if(rc) DB.checkRc(stmt.db.pointer, rc); - stmt._mayGet = false; return stmt; }; @@ -1620,16 +1733,23 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ This method always throws if called when it is illegal to do so. Namely, when triggered via a per-row callback handler of a DB.exec() call. + + If Stmt does not own its underlying (sqlite3_stmt*) (see + Stmt.wrapHandle()) then this function will not pass it to + sqlite3_finalize(). */ finalize: function(){ - if(this.pointer){ + const ptr = this.pointer; + if(ptr){ affirmNotLockedByExec(this,'finalize()'); - const rc = capi.sqlite3_finalize(this.pointer); - delete __stmtMap.get(this.db)[this.pointer]; + const rc = (__doesNotOwnHandle.delete(this) + ? 0 + : capi.sqlite3_finalize(ptr)); + delete __stmtMap.get(this.db)[ptr]; __ptrMap.delete(this); - delete this._mayGet; + __execLock.delete(this); + __stmtMayGet.delete(this); delete this.parameterCount; - delete this._lockedByExec; delete this.db; return rc; } @@ -1643,7 +1763,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ clearBindings: function(){ affirmNotLockedByExec(affirmStmtOpen(this), 'clearBindings()') capi.sqlite3_clear_bindings(this.pointer); - this._mayGet = false; + __stmtMayGet.delete(this); return this; }, /** @@ -1669,7 +1789,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ affirmNotLockedByExec(this,'reset()'); if(alsoClearBinds) this.clearBindings(); const rc = capi.sqlite3_reset(affirmStmtOpen(this).pointer); - this._mayGet = false; + __stmtMayGet.delete(this); checkSqlite3Rc(this.db, rc); return this; }, @@ -1756,7 +1876,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }else if(!this.parameterCount){ toss3("This statement has no bindable parameters."); } - this._mayGet = false; + __stmtMayGet.delete(this); if(null===arg){ /* bind NULL */ return bindOne(this, ndx, BindTypes.null, arg); @@ -1821,14 +1941,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ affirmNotLockedByExec(this, 'step()'); const rc = capi.sqlite3_step(affirmStmtOpen(this).pointer); switch(rc){ - case capi.SQLITE_DONE: return this._mayGet = false; - case capi.SQLITE_ROW: return this._mayGet = true; - default: - this._mayGet = false; - sqlite3.config.warn("sqlite3_step() rc=",rc, - capi.sqlite3_js_rc_str(rc), - "SQL =", capi.sqlite3_sql(this.pointer)); - DB.checkRc(this.db.pointer, rc); + case capi.SQLITE_DONE: + __stmtMayGet.delete(this); + return false; + case capi.SQLITE_ROW: + __stmtMayGet.add(this); + return true; + default: + __stmtMayGet.delete(this); + sqlite3.config.warn("sqlite3_step() rc=",rc, + capi.sqlite3_js_rc_str(rc), + "SQL =", capi.sqlite3_sql(this.pointer)); + DB.checkRc(this.db.pointer, rc); } }, /** @@ -1913,7 +2037,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ getJSON() can be used for that. */ get: function(ndx,asType){ - if(!affirmStmtOpen(this)._mayGet){ + if(!__stmtMayGet.has(affirmStmtOpen(this))){ toss3("Stmt.step() has not (recently) returned true."); } if(Array.isArray(ndx)){ @@ -2109,6 +2233,64 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ set: ()=>toss3("The columnCount property is read-only.") }); + Object.defineProperty(Stmt.prototype, 'parameterCount', { + enumerable: false, + get: function(){return capi.sqlite3_bind_parameter_count(this.pointer)}, + set: ()=>toss3("The parameterCount property is read-only.") + }); + + /** + The Stmt counterpart of oo1.DB.wrapHandle(), this creates a Stmt + instance which wraps a WASM (sqlite3_stmt*) in the oo1 API, + optionally with or without taking over ownership of that pointer. + + The first argument must be an oo1.DB instance[^1]. + + The second argument must be a valid WASM (sqlite3_stmt*), as + produced by sqlite3_prepare_v2() and sqlite3_prepare_v3(). + + The third argument, defaulting to false, specifies whether the + returned Stmt object takes over ownership of the underlying + (sqlite3_stmt*). If true, the returned object's finalize() method + will finalize that handle, else it will not. If it is false, + ownership of pStmt is unchanged and pStmt MUST outlive the + returned object or results are undefined. + + This function throws if the arguments are invalid. On success it + returns a new Stmt object which wraps the given statement + pointer. + + Like all Stmt objects, the finalize() method must eventually be + called on the returned object to free up internal resources, + regardless of whether this function's third argument is true or + not. + + [^1]: The first argument cannot be a (sqlite3*) because the + resulting Stmt object requires a parent DB object. It is not yet + determined whether it would be of general benefit to refactor the + DB/Stmt pair internals to communicate in terms of the underlying + (sqlite3*) rather than a DB object. If so, we could laxen the + first argument's requirement and allow an (sqlite3*). Because + DB.wrapHandle() enables multiple DB objects to proxy the same + (sqlite3*), we cannot unambiguously translate the first arugment + from (sqlite3*) to DB instances for us with this function's first + argument. + */ + Stmt.wrapHandle = function(oo1db, pStmt, takeOwnership=false){ + let ctor = Stmt; + if( !(oo1db instanceof DB) || !oo1db.pointer ){ + throw new sqlite3.SQLite3Error(sqlite3.SQLITE_MISUSE, + "First argument must be an opened "+ + "sqlite3.oo1.DB instance"); + } + if( !pStmt || !wasm.isPtr(pStmt) ){ + throw new sqlite3.SQLite3Error(sqlite3.SQLITE_MISUSE, + "Second argument must be a WASM "+ + "sqlite3_stmt pointer"); + } + return new Stmt(oo1db, pStmt, BindTypes, !!takeOwnership); + } + /** The OO API's public namespace. */ sqlite3.oo1 = { DB, diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 7e128a3fa..e3807a314 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -134,22 +134,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( const config = Object.assign(Object.create(null),{ exports: undefined, memory: undefined, - bigIntEnabled: (()=>{ - if('undefined'!==typeof Module){ - /* Emscripten module will contain HEAPU64 when built with - -sWASM_BIGINT=1, else it will not. - - As of emsdk 3.1.55, when building in strict mode, HEAPxyz - are only available if _explicitly_ included in the exports, - else they are not. We do not (as of 2024-03-04) use -sSTRICT - for the canonical builds. - */ - if( !!Module.HEAPU64 ) return true; - /* Else fall through and hope for the best. Nobody _really_ - builds this without BigInt support, do they? */ - } - return !!globalThis.BigInt64Array; - })(), + bigIntEnabled: !!globalThis.BigInt64Array, debug: console.debug.bind(console), warn: console.warn.bind(console), error: console.error.bind(console), diff --git a/ext/wasm/api/sqlite3-api-worker1.c-pp.js b/ext/wasm/api/sqlite3-api-worker1.c-pp.js index 5e088f438..55ad16185 100644 --- a/ext/wasm/api/sqlite3-api-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-worker1.c-pp.js @@ -385,10 +385,19 @@ sqlite3.initWorker1API = function(){ const getDbId = function(db){ let id = wState.idMap.get(db); if(id) return id; - id = 'db#'+(++wState.idSeq)+'@'+db.pointer; + id = 'db#'+(++wState.idSeq)+':'+ + Math.floor(Math.random() * 100000000)+':'+ + Math.floor(Math.random() * 100000000); /** ^^^ can't simply use db.pointer b/c closing/opening may re-use the same address, which could map pending messages to a wrong - instance. */ + instance. + + 2025-07: https://github.com/sqlite/sqlite-wasm/issues/113 + demonstrates that two Worker1s can end up with the same IDs, + despite using different instances of the library, so we need + to add some randomness to the IDs instead of relying on the + pointer addresses. + */ wState.idMap.set(db, id); return id; }; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 1850d313c..574684ce9 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -135,9 +135,12 @@ /* ** If SQLITE_WASM_BARE_BONES is defined, undefine most of the ENABLE -** macros. +** macros. This will, when using the canonical makefile, also elide +** any C functions from the WASM exports which are listed in +** ./EXPORT_FUNCTIONS.sqlite3-extras. */ #ifdef SQLITE_WASM_BARE_BONES +# undef SQLITE_ENABLE_COLUMN_METADATA # undef SQLITE_ENABLE_DBPAGE_VTAB # undef SQLITE_ENABLE_DBSTAT_VTAB # undef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -936,6 +939,7 @@ const char * sqlite3__wasm_enum_json(void){ DefInt(SQLITE_INNOCUOUS); DefInt(SQLITE_SUBTYPE); DefInt(SQLITE_RESULT_SUBTYPE); + DefInt(SQLITE_SELFORDER1); } _DefGroup; DefGroup(version) { diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index c043fd148..2edabe3e6 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -130,12 +130,10 @@ Notable shortcomings: - - This API was not designed with ES6 modules in mind. Neither Firefox - nor Safari support, as of March 2023, the {type:"module"} flag to the - Worker constructor, so that particular usage is not something we're going - to target for the time being: - - https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker + - "v1" of this this API is not suitable for use as an ESM module + because ESM worker modules were not widely supported when it was + developed. For use as an ESM module, see the "v2" interface later + on in this file. */ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ // Inspired by: https://stackoverflow.com/a/52439530 @@ -296,7 +294,7 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { after calling the original function and will reject if that function throws. */ -sqlite3Worker1Promiser.v2 = function(config){ +globalThis.sqlite3Worker1Promiser.v2 = function callee(config = callee.defaultConfig){ let oldFunc; if( 'function' == typeof config ){ oldFunc = config; @@ -326,11 +324,14 @@ sqlite3Worker1Promiser.v2 = function(config){ } return p; }.bind({ - /* We do this because clients are - recommended to delete globalThis.sqlite3Worker1Promiser. */ + /* We do this because clients are recommended to delete + globalThis.sqlite3Worker1Promiser. */ original: sqlite3Worker1Promiser }); +globalThis.sqlite3Worker1Promiser.v2.defaultConfig = + globalThis.sqlite3Worker1Promiser.defaultConfig; + //#if target=es6-module /** When built as a module, we export sqlite3Worker1Promiser.v2() diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index b4d8f691b..e85669579 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -1773,10 +1773,10 @@ globalThis.WhWasmUtilInstaller = function(target){ does not have a stable interface. */ xArg.FuncPtrAdapter.warnOnUse = false; - /** If true, convertArg() will FuncPtrAdapter.debugOut() when it - (un)installs a function binding to/from WASM. Note that - deinstallation of bindScope=transient bindings happens - via scopedAllocPop() so will not be output. */ + /** If true, convertArg() will call FuncPtrAdapter.debugOut() when + it (un)installs a function binding to/from WASM. Note that + deinstallation of bindScope=transient bindings happens via + scopedAllocPop() so will not be output. */ xArg.FuncPtrAdapter.debugFuncInstall = false; /** Function used for debug output. */ @@ -1827,9 +1827,8 @@ globalThis.WhWasmUtilInstaller = function(target){ The first argument must be one of: - A JavaScript function. - - The name of a WASM-exported function. In the latter case xGet() - is used to fetch the exported function, which throws if it's not - found. + - The name of a WASM-exported function. xGet() is used to fetch + the exported function, which throws if it's not found. - A pointer into the indirect function table. e.g. a pointer returned from target.installFunction(). @@ -1874,9 +1873,6 @@ globalThis.WhWasmUtilInstaller = function(target){ which convert their argument to an integer and truncate it to the given bit length. - - `N*` (args): a type name in the form `N*`, where N is a numeric - type name, is treated the same as WASM pointer. - - `*` and `pointer` (args): are assumed to be WASM pointer values and are returned coerced to an appropriately-sized pointer value (i32 or i64). Non-numeric values will coerce to 0 and @@ -1887,7 +1883,15 @@ globalThis.WhWasmUtilInstaller = function(target){ WASM pointer numeric type. - `**` (args): is simply a descriptive alias for the WASM pointer - type. It's primarily intended to mark output-pointer arguments. + type. It's primarily intended to mark output-pointer arguments, + noting that JS's view of WASM does not distinguish between + pointers and pointers-to-pointers, so all such interpretation + of `**`, as distinct from `*`, necessarily happens at the + client level. + + - `NumType*` (args): a type name in this form, where T is + the name of a numeric mapping, e.g. 'int16' or 'double', + is treated like `*`. - `i64` (args and results): passes the value to BigInt() to convert it to an int64. Only available if bigIntEnabled is @@ -1916,7 +1920,7 @@ globalThis.WhWasmUtilInstaller = function(target){ UTF-8-encoded C-string to pass to the exported function, cleaning it up before the wrapper returns. If a long-lived C-string pointer is required, that requires client-side code - to create the string, then pass its pointer to the function. + to create the string then pass its pointer to the function. - Else the arg is assumed to be a pointer to a string the client has already allocated and it's passed on as @@ -2091,8 +2095,8 @@ globalThis.WhWasmUtilInstaller = function(target){ easily convert, e.g., to C-strings, and have them cleaned up automatically before the wrapper returns to the caller. Likewise, if a _result_ adapter uses scoped allocation, the result will be - freed before because they would be freed before the wrapper - returns, leading to chaos and undefined behavior. + freed before the wrapper returns, leading to chaos and undefined + behavior. Except when called as a getter, this function returns itself. */ diff --git a/ext/wasm/config.make.in b/ext/wasm/config.make.in index f30baac3f..4c8d7893b 100644 --- a/ext/wasm/config.make.in +++ b/ext/wasm/config.make.in @@ -1,15 +1,16 @@ -# Gets filtered by the configure script +# config.make.in gets filtered by the top-most configure script to +# create config.make. bin.bash = @BIN_BASH@ bin.emcc = @EMCC_WRAPPER@ bin.wasm-strip = @BIN_WASM_STRIP@ bin.wasm-opt = @BIN_WASM_OPT@ -SHELL := $(bin.bash) +SHELL = $(bin.bash) # The following overrides can be uncommented to test various # validation and if/else branches the makefile code: # -#bin.bash := -#bin.emcc := -#bin.wasm-strip := -#bin.wasm-opt := +#bin.bash = +#bin.emcc = +#bin.wasm-strip = +#bin.wasm-opt = diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index 60699ff5c..176972fd7 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -11,7 +11,7 @@ # distinctly different zip file and top directory name to distinguish # them from release builds. ####################################################################### -MAKEFILE.dist := $(lastword $(MAKEFILE_LIST)) +MAKEFILE.dist = $(lastword $(MAKEFILE_LIST)) ######################################################################## # Chicken/egg situation: we need $(bin.version-info) to get the @@ -20,16 +20,16 @@ MAKEFILE.dist := $(lastword $(MAKEFILE_LIST)) # have to use a temporary name for the archive until we can get # that binary built. ifeq (1,$(SQLITE_C_IS_SEE)) -dist-name-extra := -see +dist-name-extra = -see else -dist-name-extra := +dist-name-extra = endif ifeq (,$(filter snapshot,$(MAKECMDGOALS))) -dist-name-prefix := sqlite-wasm$(dist-name-extra) +dist-name-prefix = sqlite-wasm$(dist-name-extra) else -dist-name-prefix := sqlite-wasm$(dist-name-extra)-snapshot-$(shell /usr/bin/date +%Y%m%d) +dist-name-prefix = sqlite-wasm$(dist-name-extra)-snapshot-$(shell /usr/bin/date +%Y%m%d) endif -dist-name := $(dist-name-prefix)-TEMP +dist-name = $(dist-name-prefix)-TEMP ######################################################################## # dist.build must be the name of a target which triggers the build of @@ -45,10 +45,10 @@ dist-name := $(dist-name-prefix)-TEMP # reason not to. dist.build ?= oz -dist-dir.top := $(dist-name) -dist-dir.jswasm := $(dist-dir.top)/$(notdir $(dir.dout)) -dist-dir.common := $(dist-dir.top)/common -dist.top.extras := \ +dist-dir.top = $(dist-name) +dist-dir.jswasm = $(dist-dir.top)/$(notdir $(dir.dout)) +dist-dir.common = $(dist-dir.top)/common +dist.top.extras = \ demo-123.html demo-123-worker.html demo-123.js \ tester1.html tester1-worker.html tester1-esm.html \ tester1.js tester1.mjs \ @@ -56,9 +56,9 @@ dist.top.extras := \ demo-worker1.html demo-worker1.js \ demo-worker1-promiser.html demo-worker1-promiser.js \ demo-worker1-promiser-esm.html demo-worker1-promiser.mjs -dist.jswasm.extras := $(sqlite3.wasm) \ +dist.jswasm.extras = $(sqlite3.wasm) \ $(sqlite3-api.ext.jses) -dist.common.extras := \ +dist.common.extras = \ $(wildcard $(dir.common)/*.css) \ $(dir.common)/SqliteTestUtil.js @@ -77,12 +77,12 @@ $(bin.stripccomments) $(2) < $(1) > $(dist-dir.jswasm)/$(notdir $(1)) || exit; endef # STRIP_K1.js = list of JS files which need to be passed through # $(bin.stripcomments) with a single -k flag. -STRIP_K1.js := $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js) \ +STRIP_K1.js = $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js) \ $(sqlite3-worker1-bundler-friendly.js) \ $(sqlite3-api.ext.jses) # STRIP_K2.js = list of JS files which need to be passed through # $(bin.stripcomments) with two -k flags. -STRIP_K2.js := $(sqlite3.js) $(sqlite3.mjs) \ +STRIP_K2.js = $(sqlite3.js) $(sqlite3.mjs) \ $(sqlite3-bundler-friendly.mjs) $(sqlite3-node.mjs) ######################################################################## # dist: create the end-user deliverable archive. @@ -104,8 +104,8 @@ STRIP_K2.js := $(sqlite3.js) $(sqlite3.mjs) \ # to know that it's in a regex or string literal. Because of that, # comment-stripping is currently disabled, which means the builds will # be significantly larger than before. -#apply_comment_stripper := false -apply_comment_stripper := true +#apply_comment_stripper = false +apply_comment_stripper = true # ^^^ shell command true or false dist: \ $(bin.stripccomments) $(bin.version-info) \ diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 8110384a6..6bdf44195 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -3,13 +3,12 @@ # # Intended to include'd by ./GNUmakefile. ####################################################################### -MAKEFILE.fiddle := $(lastword $(MAKEFILE_LIST)) ######################################################################## # shell.c and its build flags... ifneq (1,$(MAKING_CLEAN)) - make-np-0 := make -C $(dir.top) -n -p - make-np-1 := sed -e 's/(TOP)/(dir.top)/g' + make-np-0 = make -C $(dir.top) -n -p + make-np-1 = sed -e 's/(TOP)/(dir.top)/g' # Extract SHELL_OPT and SHELL_DEP from the top-most makefile and import # them as vars here... $(eval $(shell $(make-np-0) | grep -e '^SHELL_OPT ' | $(make-np-1))) @@ -27,7 +26,7 @@ endif # /shell.c ######################################################################## -EXPORTED_FUNCTIONS.fiddle := $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle +EXPORTED_FUNCTIONS.fiddle = $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle fiddle.emcc-flags = \ $(emcc.cflags) $(emcc_opt_full) \ --minify 0 \ @@ -40,9 +39,8 @@ fiddle.emcc-flags = \ -sWASM_BIGINT=$(emcc.WASM_BIGINT) \ -sEXPORT_NAME=$(sqlite3.js.init-func) \ -Wno-limited-postlink-optimizations \ - $(emcc.exportedRuntimeMethods) \ + $(emcc.exportedRuntimeMethods),FS \ -sEXPORTED_FUNCTIONS=@$(abspath $(EXPORTED_FUNCTIONS.fiddle)) \ - -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory \ $(SQLITE_OPT.full-featured) \ $(SQLITE_OPT.common) \ $(SHELL_OPT) \ @@ -53,12 +51,12 @@ fiddle.emcc-flags = \ # Flags specifically for debug builds of fiddle. Performance suffers # greatly in debug builds. -fiddle.emcc-flags.debug := $(fiddle.emcc-flags) \ +fiddle.emcc-flags.debug = $(fiddle.emcc-flags) \ -DSQLITE_DEBUG \ -DSQLITE_ENABLE_SELECTTRACE \ -DSQLITE_ENABLE_WHERETRACE -fiddle.EXPORTED_FUNCTIONS.in := \ +fiddle.EXPORTED_FUNCTIONS.in = \ EXPORTED_FUNCTIONS.fiddle.in \ $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core \ $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras @@ -67,10 +65,7 @@ $(EXPORTED_FUNCTIONS.fiddle): $(MKDIR.bld) $(fiddle.EXPORTED_FUNCTIONS.in) \ $(MAKEFILE.fiddle) sort -u $(fiddle.EXPORTED_FUNCTIONS.in) > $@ -fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.c) - -fiddle: $(fiddle-module.js) $(fiddle-module.js.debug) -fiddle.debug: $(fiddle-module.js.debug) +fiddle.cses = $(dir.top)/shell.c $(sqlite3-wasm.c) clean: clean-fiddle clean-fiddle: @@ -84,10 +79,9 @@ clean-fiddle: all: fiddle ######################################################################## -# fiddle_remote is the remote destination for the fiddle app. It -# must be a [user@]HOST:/path for rsync. -# Note that the target "should probably" contain a symlink of -# index.html -> fiddle.html. +# fiddle_remote is the remote destination for the fiddle app. It must +# be a [user@]HOST:/path for rsync. The target "should probably" +# contain a symlink of index.html -> fiddle.html. fiddle_remote ?= ifeq (,$(fiddle_remote)) ifneq (,$(wildcard /home/stephan)) @@ -154,11 +148,6 @@ push-fiddle: fiddle # because certain execution environments disallow those constructs. # This flag is not strictly necessary, however. # -# -sWASM_BIGINT is UNTESTED but "should" allow the int64-using C APIs -# to work with JS/wasm, insofar as the JS environment supports the -# BigInt type. That support requires an extremely recent browser: -# Safari didn't get that support until late 2020. -# # --no-entry: for compiling library code with no main(). If this is # not supplied and the code has a main(), it is called as part of the # module init process. Note that main() is #if'd out of shell.c @@ -180,14 +169,15 @@ push-fiddle: fiddle # minification makes little difference in terms of overall # distributable size. # -# --minify 0: disables minification of the generated JS code, -# regardless of optimization level. Minification of the JS has -# minimal overall effect in the larger scheme of things and results -# in JS files which can neither be edited nor viewed as text files in -# Fossil (which flags them as binary because of their extreme line -# lengths). Interestingly, whether or not the comments in the -# generated JS file get stripped is unaffected by this setting and -# depends entirely on the optimization level. Higher optimization +# --minify 0: supposedly disables minification of the generated JS +# code, regardless of optimization level, but that's not quite true: +# search the main makefile for wasm-strip for details. Minification +# of the JS has minimal overall effect in the larger scheme of things +# and results in JS files which can neither be edited nor viewed as +# text files in Fossil (which flags them as binary because of their +# extreme line lengths). Interestingly, whether or not the comments +# in the generated JS file get stripped is unaffected by this setting +# and depends entirely on the optimization level. Higher optimization # levels reduce the size of the JS considerably even without # minification. # diff --git a/ext/wasm/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js index 27d915eb2..9c6cddb0f 100644 --- a/ext/wasm/fiddle/fiddle-worker.js +++ b/ext/wasm/fiddle/fiddle-worker.js @@ -163,9 +163,11 @@ fiddleModule.isDead = true; return false; } - stdout("SQLite version", capi.sqlite3_libversion(), - capi.sqlite3_sourceid().substr(0,19)); - stdout('Welcome to the "fiddle" shell.'); + wMsg('sqlite-version', { + lib: capi.sqlite3_libversion(), + srcId: capi.sqlite3_sourceid() + }); + stdout('Welcome to the "fiddle" shell. Tap the About button for more info.'); if(capi.sqlite3_vfs_find("opfs")){ stdout("\nOPFS is available. To open a persistent db, use:\n\n", " .open file:name?vfs=opfs\n\nbut note that some", @@ -281,6 +283,7 @@ stderr("'open' expects {buffer:Uint8Array} containing an uploaded db."); return; } + buffer.set([1,1], 18)/*force db out of WAL mode*/; const fn = ( opt.filename ? opt.filename.split(/[/\\]/).pop().replace('"','_') diff --git a/ext/wasm/fiddle/fiddle.js b/ext/wasm/fiddle/fiddle.js index f0a89f25d..877a87772 100644 --- a/ext/wasm/fiddle/fiddle.js +++ b/ext/wasm/fiddle/fiddle.js @@ -329,6 +329,21 @@ SF.worker = new Worker('fiddle-worker.js'+self.location.search); SF.worker.onmessage = (ev)=>SF.runMsgHandlers(ev.data); SF.addMsgHandler(['stdout', 'stderr'], (ev)=>SF.echo(ev.data)); + SF.addMsgHandler('sqlite-version', (ev)=>{ + const v = ev.data; + const a = E('#sqlite-version-link'); + const li = v.srcId.split(' ')/*DATE TIME HASH*/; + a.setAttribute('href', + //'https://sqlite.org/src/timeline/?c='+li[2].substr(0,20) + 'https://sqlite.org/src/info/'+li[2].substr(0,20) + ); + a.setAttribute('target', '_blank'); + a.innerText = [ + v.lib, + v.srcId.substr(0,34) + ].join(' '); + SF.echo("SQLite version",a.innerText); + }); /* querySelectorAll() proxy */ const EAll = function(/*[element=document,] cssSelector*/){ @@ -391,6 +406,143 @@ self.onSFLoaded(); }); + SF.e ={ + about: E('#view-about'), + split: E('#view-split'), + terminal: E('#view-terminal'), + hideInTerminal: EAll('.hide-in-terminal' + /* Elements to hide when in terminal mode */) + }; + SF.eViews = EAll('.app-view'); + SF.setMainView = function(eMain){ + if( SF.e.main === eMain ) return; + SF.eViews.forEach((e)=>{ + if( e===eMain ) e.classList.remove('hidden'); + else e.classList.add('hidden'); + }); + SF.e.hideInTerminal.forEach(e=>{ + if( eMain === SF.e.terminal ) e.classList.add('hidden'); + else e.classList.remove('hidden'); + }); + SF.e.main = eMain; + SF.ForceResizeKludge(); + }; + + /** Toggle the "About" view on and off. */ + SF.toggleAbout = function(){ + if( SF.e.about.classList.contains('hidden') ){ + SF.e.about.$returnTo = SF.e.main; + SF.setMainView( SF.e.about ); + }else{ + const e = SF.e.about.$returnTo; + delete SF.e.about.$returnTo; + SF.setMainView( e ); + } + }; + + /** + Given a DOM element, this routine measures its "effective + height", which is the bounding top/bottom range of this element + and all of its children, recursively. For some DOM structure + cases, a parent may have a reported height of 0 even though + children have non-0 sizes. + + Returns 0 if !e or if the element really has no height. + */ + const effectiveHeight = function f(e){ + if(!e) return 0; + if(!f.measure){ + f.measure = function callee(e, depth){ + if(!e) return; + const m = e.getBoundingClientRect(); + if(0===depth){ + callee.top = m.top; + callee.bottom = m.bottom; + }else{ + callee.top = m.top ? Math.min(callee.top, m.top) : callee.top; + callee.bottom = Math.max(callee.bottom, m.bottom); + } + Array.prototype.forEach.call(e.children,(e)=>callee(e,depth+1)); + if(0===depth){ + //console.debug("measure() height:",e.className, callee.top, callee.bottom, (callee.bottom - callee.top)); + f.extra += callee.bottom - callee.top; + } + return f.extra; + }; + } + f.extra = 0; + f.measure(e,0); + return f.extra; + }; + + /** + Returns a function, that, as long as it continues to be invoked, + will not be triggered. The function will be called after it stops + being called for N milliseconds. If `immediate` is passed, call + the callback immediately and hinder future invocations until at + least the given time has passed. + + If passed only 1 argument, or passed a falsy 2nd argument, + the default wait time set in this function's $defaultDelay + property is used. + + Source: underscore.js, by way of https://davidwalsh.name/javascript-debounce-function + */ + const debounce = function f(func, wait, immediate) { + var timeout; + if(!wait) wait = f.$defaultDelay; + return function() { + const context = this, args = Array.prototype.slice.call(arguments); + const later = function() { + timeout = undefined; + if(!immediate) func.apply(context, args); + }; + const callNow = immediate && !timeout; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + if(callNow) func.apply(context, args); + }; + }; + debounce.$defaultDelay = 500 /*arbitrary*/; + + SF.ForceResizeKludge = (function(){ + /* Workaround for Safari mayhem regarding use of vh CSS + units.... We cannot use vh units to set the main view + size because Safari chokes on that, so we calculate + that height here. Larger than ~95% is too big for + Firefox on Android, causing the input area to move + off-screen. */ + const eVisibles = EAll('.app-view'); + const elemsToCount = [ + /* Elements which we need to always count in the + visible body size. */ + E('body > header'), + E('body > footer'), + E('body > fieldset.options') + ]; + const resized = function f(){ + if(f.$disabled) return; + const wh = window.innerHeight; + var ht; + var extra = 0; + elemsToCount.forEach((e)=>e ? extra += effectiveHeight(e) : false); + ht = wh - extra; + eVisibles.forEach(function(e){ + e.style.height = + e.style.maxHeight = [ + "calc(", (ht>=100 ? ht : 100), "px", + " - 2em"/*fudge value*/,")" + /* ^^^^ hypothetically not needed, but both + Chrome/FF on Linux will force scrollbars on the + body if this value is too small. */ + ].join(''); + }); + }; + resized.$disabled = true/*gets deleted when setup is finished*/; + window.addEventListener('resize', debounce(resized, 250), false); + return resized; + })(); + /** Performs all app initialization which must wait until after the worker module is loaded. This function removes itself when it's @@ -398,9 +550,19 @@ */ self.onSFLoaded = function(){ delete this.onSFLoaded; + // Unhide all elements which start out hidden EAll('.initially-hidden').forEach((e)=>e.classList.remove('initially-hidden')); + SF.e.main = EAll('.app-view:not(.hidden)')[0] + /** The main view widget. Initially the first non-hidden + .app-view element. */; + if( (new URL(self.location.href).searchParams).has('about') ){ + SF.toggleAbout() /* for use while editing the About page */; + } E('#btn-reset').addEventListener('click',()=>SF.resetDb()); + EAll('#btn-about, #btn-about-close').forEach((e)=>{ + e.addEventListener('click',()=>SF.toggleAbout()) + }); const taInput = E('#input'); const btnClearIn = E('#btn-clear'); const selectExamples = E('#select-examples'); @@ -618,108 +780,6 @@ }, false); }); - /** - Given a DOM element, this routine measures its "effective - height", which is the bounding top/bottom range of this element - and all of its children, recursively. For some DOM structure - cases, a parent may have a reported height of 0 even though - children have non-0 sizes. - - Returns 0 if !e or if the element really has no height. - */ - const effectiveHeight = function f(e){ - if(!e) return 0; - if(!f.measure){ - f.measure = function callee(e, depth){ - if(!e) return; - const m = e.getBoundingClientRect(); - if(0===depth){ - callee.top = m.top; - callee.bottom = m.bottom; - }else{ - callee.top = m.top ? Math.min(callee.top, m.top) : callee.top; - callee.bottom = Math.max(callee.bottom, m.bottom); - } - Array.prototype.forEach.call(e.children,(e)=>callee(e,depth+1)); - if(0===depth){ - //console.debug("measure() height:",e.className, callee.top, callee.bottom, (callee.bottom - callee.top)); - f.extra += callee.bottom - callee.top; - } - return f.extra; - }; - } - f.extra = 0; - f.measure(e,0); - return f.extra; - }; - - /** - Returns a function, that, as long as it continues to be invoked, - will not be triggered. The function will be called after it stops - being called for N milliseconds. If `immediate` is passed, call - the callback immediately and hinder future invocations until at - least the given time has passed. - - If passed only 1 argument, or passed a falsy 2nd argument, - the default wait time set in this function's $defaultDelay - property is used. - - Source: underscore.js, by way of https://davidwalsh.name/javascript-debounce-function - */ - const debounce = function f(func, wait, immediate) { - var timeout; - if(!wait) wait = f.$defaultDelay; - return function() { - const context = this, args = Array.prototype.slice.call(arguments); - const later = function() { - timeout = undefined; - if(!immediate) func.apply(context, args); - }; - const callNow = immediate && !timeout; - clearTimeout(timeout); - timeout = setTimeout(later, wait); - if(callNow) func.apply(context, args); - }; - }; - debounce.$defaultDelay = 500 /*arbitrary*/; - - const ForceResizeKludge = (function(){ - /* Workaround for Safari mayhem regarding use of vh CSS - units.... We cannot use vh units to set the main view - size because Safari chokes on that, so we calculate - that height here. Larger than ~95% is too big for - Firefox on Android, causing the input area to move - off-screen. */ - const appViews = EAll('.app-view'); - const elemsToCount = [ - /* Elements which we need to always count in the - visible body size. */ - E('body > header'), - E('body > footer') - ]; - const resized = function f(){ - if(f.$disabled) return; - const wh = window.innerHeight; - var ht; - var extra = 0; - elemsToCount.forEach((e)=>e ? extra += effectiveHeight(e) : false); - ht = wh - extra; - appViews.forEach(function(e){ - e.style.height = - e.style.maxHeight = [ - "calc(", (ht>=100 ? ht : 100), "px", - " - 2em"/*fudge value*/,")" - /* ^^^^ hypothetically not needed, but both - Chrome/FF on Linux will force scrollbars on the - body if this value is too small. */ - ].join(''); - }); - }; - resized.$disabled = true/*gets deleted when setup is finished*/; - window.addEventListener('resize', debounce(resized, 250), false); - return resized; - })(); - /** Set up a selection list of examples */ (function(){ const xElem = E('#select-examples'); @@ -790,33 +850,35 @@ })()/* example queries */; //SF.echo(null/*clear any output generated by the init process*/); - if(window.jQuery && window.jQuery.terminal){ + if(window.jQuery && window.jQuery.terminal && SF.e.terminal){ /* Set up the terminal-style view... */ - const eTerm = window.jQuery('#view-terminal').empty(); - SF.jqTerm = eTerm.terminal(SF.dbExec.bind(SF),{ + const jqeTerm = window.jQuery(SF.e.terminal).empty(); + SF.jqTerm = jqeTerm.terminal(SF.dbExec.bind(SF),{ prompt: 'sqlite> ', greetings: false /* note that the docs incorrectly call this 'greeting' */ }); + EAll('.unhide-if-terminal-available').forEach(e=>{ + e.classList.remove('hidden'); + }); + EAll('.remove-if-terminal-available').forEach(e=>{ + e.parentElement.removeChild(e); + }); /* Set up a button to toggle the views... */ - const head = E('header#titlebar'); + const ePlaceholder = E('#terminal-button-placeholder'); + ePlaceholder.classList.add('labeled-input'); + ePlaceholder.classList.remove('hidden'); const btnToggleView = document.createElement('button'); - btnToggleView.appendChild(document.createTextNode("Toggle View")); - head.appendChild(btnToggleView); + btnToggleView.innerText = "Toggle view"; + ePlaceholder.appendChild( btnToggleView ); btnToggleView.addEventListener('click',function f(){ - EAll('.app-view').forEach(e=>e.classList.toggle('hidden')); - if(document.body.classList.toggle('terminal-mode')){ - ForceResizeKludge(); - } + const terminalOn = document.body.classList.toggle('terminal-mode'); + SF.setMainView( terminalOn ? SF.e.terminal : SF.e.split ); }, false); btnToggleView.click()/*default to terminal view*/; } - SF.echo('This experimental app is provided in the hope that it', - 'may prove interesting or useful but is not an officially', - 'supported deliverable of the sqlite project. It is subject to', - 'any number of changes or outright removal at any time.\n'); const urlParams = new URL(self.location.href).searchParams; SF.dbExec(urlParams.get('sql') || null); - delete ForceResizeKludge.$disabled; - ForceResizeKludge(); + delete SF.ForceResizeKludge.$disabled; + SF.ForceResizeKludge(); }/*onSFLoaded()*/; })(); diff --git a/ext/wasm/fiddle/index.html b/ext/wasm/fiddle/index.html index ca6788ef0..7f79b754b 100644 --- a/ext/wasm/fiddle/index.html +++ b/ext/wasm/fiddle/index.html @@ -5,16 +5,27 @@ <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> <title>SQLite3 Fiddle</title> <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> - <!-- to add a toggleable terminal-style view, uncomment the following - two lines and ensure that these files are on the web server. --> + <!-- + To add a terminal-style view using jquery.terminal[^1], + uncomment the following two HTML lines and ensure that these + files are on the web server. + + jquery-bundle.min.js is a concatenation of jquery.min.js from + [^2] and jquery.terminal.min.js from [^1]. + jquery.terminal.min.css is from [^1]. + + [^1]: https://github.com/jcubic/jquery.terminal + [^2]: https://jquery.com + --> <!--script src="jqterm/jqterm-bundle.min.js"></script> - <link rel="stylesheet" href="jqterm/jquery.terminal.min.css"/--> + <link rel="stylesheet" href="jqterm/jquery.terminal.min.css"--> <style> /* The following styles are for app-level use. */ :root { --sqlite-blue: #044a64; - --textarea-color1: #044a64; + --textarea-color1: #000 /*044a64 is nice too*/; --textarea-color2: white; + --size: 1.25 /* used by jqterm to calculate font size and the default is too tiny.*/; } textarea { font-family: monospace; @@ -170,6 +181,17 @@ display: flex; flex-direction: column-reverse; } + #view-about { + flex: auto; + overflow: auto; + } + #view-about h1:first-child { + display: flex; + } + #view-about h1:first-child > button { + align-self: center; + margin-left: 1em; + } /* emcscript-related styling, used during the module load/intialization processes... */ .emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; } @@ -200,8 +222,10 @@ <body> <header id='titlebar'> <span>SQLite3 Fiddle</span> - <span class='powered-by'>Powered by - <a href='https://sqlite.org'>SQLite3</a></span> + <span id='titlebar-buttons'> + <span class='powered-by'>Powered by + <a href='https://sqlite.org'>SQLite3</a></span> + </span> </header> <!-- emscripten bits --> <figure id="module-spinner"> @@ -215,7 +239,7 @@ </figure> <div class="emscripten" id="module-status">Downloading...</div> <div class="emscripten"> - <progress value="0" max="100" id="module-progress" hidden='1'></progress> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> </div><!-- /emscripten bits --> <div id='view-terminal' class='app-view hidden initially-hidden'> @@ -224,45 +248,6 @@ </div> <div id='view-split' class='app-view initially-hidden'> - <fieldset class='options collapsible'> - <legend><button class='fieldset-toggle'>Options</button></legend> - <div class=''> - <span class='labeled-input'> - <input type='checkbox' id='opt-cb-sbs' - data-csstgt='#main-wrapper' - data-cssclass='side-by-side' - data-config='sideBySide'> - <label for='opt-cb-sbs'>Side-by-side</label> - </span> - <span class='labeled-input'> - <input type='checkbox' id='opt-cb-swapio' - data-csstgt='#main-wrapper' - data-cssclass='swapio' - data-config='swapInOut'> - <label for='opt-cb-swapio'>Swap in/out</label> - </span> - <span class='labeled-input'> - <input type='checkbox' id='opt-cb-autoscroll' - data-config='autoScrollOutput'> - <label for='opt-cb-autoscroll'>Auto-scroll output</label> - </span> - <span class='labeled-input'> - <input type='checkbox' id='opt-cb-autoclear' - data-config='autoClearOutput'> - <label for='opt-cb-autoclear'>Auto-clear output</label> - </span> - <span class='labeled-input'> - <input type='file' id='load-db' class='hidden'/> - <button id='btn-load-db'>Load DB...</button> - </span> - <span class='labeled-input'> - <button id='btn-export'>Download DB</button> - </span> - <span class='labeled-input'> - <button id='btn-reset'>Reset DB</button> - </span> - </div> - </fieldset><!-- .options --> <div id='main-wrapper' class=''> <fieldset class='zone-wrapper input'> <legend><div class='button-bar'> @@ -295,8 +280,113 @@ <div><textarea id="output" readonly placeholder="Shell output."></textarea></div> </fieldset> - </div> + </div><!-- #main-wrapper --> </div> <!-- #view-split --> - <script src="fiddle.js"></script> + +<div class='hidden app-view' id='view-about'> + <h1>About SQLite Fiddle <button id='btn-about-close'>close</button></h1> + + <p>Fiddle is a JavaScript application wrapping a <a href='https://webassembly.org'>WebAssembly</a> + build of <a href="https://sqlite.org/cli.html">the SQLite CLI shell</a>, slightly + modified to account for browser-based user input. Aside from the different layout, + it works just like the CLI shell. This copy was built with SQLite version + <a id='sqlite-version-link'></a>. + </p> + + <p>This app is provided in the hope that it may prove interesting or useful + but it is not an officially-supported deliverable of the SQLite project. + It is subject to any number of changes or outright removal at any time. + That said, for as long as it's online we do respond to support requests + in <a href="https://sqlite.org/forum">the SQLite forum</a>. + </p> + + <p>This app runs on your device. After loading, it does not interact + with the remote server at all. Similarly, this app does not use any + HTTP cookies.</p> + + <p>Fiddle databases are transient in-memory databases unless they + specifically use a persistent storage option (if available, help + text in the SQL result output area will indicate how to use + persistent storage when this app starts up). + </p> + + <h1>Usage Summary</h1> + + <ul> + <li class='hidden unhide-if-terminal-available'>In "terminal + mode" it accepts input just like the CLI shell does.</li> + <li>In split-view mode: + <ul> + <li>Input can be executed with either the Run + button or tapping one of Ctrl-enter or Shift-enter from within + the text input field. If a portion of the input field is + selected, only that portion will be run. + </li> + <li>The various toggle checkboxes can be used to tweak the layout + and behaviors. Those toggles are persistent if the JS environment + allows it. + </li> + </ul> + </li> + <li class='remove-if-terminal-available'>"Terminal mode" is + not available in this deployment. + </li> + <li>Databases can be imported and exported using the buttons in + the Options toolbar. No specific limit for imported database + sizes is imposed, but large databases may cause it to fail with + an out-of-memory error.</li> + <!--li></li--> + </ul> + +</div><!-- #view-about --> + +<fieldset class='options'> + <legend>Options</legend> + <div class=''> + <span class='labeled-input'> + <input type='file' id='load-db' class='hidden'/> + <button id='btn-load-db'>Load DB...</button> + </span> + <span class='labeled-input'> + <button id='btn-export'>Download DB</button> + </span> + <span class='labeled-input'> + <button id='btn-reset'>Reset DB</button> + </span> + <span id='terminal-button-placeholder' class='hidden'></span> + <span class='labeled-input'> + <button id='btn-about'>About...</button> + </span> + <span class='labeled-input hide-in-terminal'> + <input type='checkbox' id='opt-cb-sbs' + data-csstgt='#main-wrapper' + data-cssclass='side-by-side' + data-config='sideBySide' + > + <label for='opt-cb-sbs'>Side-by-side</label> + </span> + <span class='labeled-input hide-in-terminal'> + <input type='checkbox' id='opt-cb-swapio' + data-csstgt='#main-wrapper' + data-cssclass='swapio' + data-config='swapInOut' + > + <label for='opt-cb-swapio'>Swap in/out</label> + </span> + <span class='labeled-input hide-in-terminal'> + <input type='checkbox' id='opt-cb-autoscroll' + data-config='autoScrollOutput' + > + <label for='opt-cb-autoscroll'>Auto-scroll output</label> + </span> + <span class='labeled-input hide-in-terminal'> + <input type='checkbox' id='opt-cb-autoclear' + data-config='autoClearOutput'> + <label for='opt-cb-autoclear'>Auto-clear output</label> + </span> + </div> +</fieldset><!-- .options --> + +<script src="fiddle.js"></script> </body> </html> diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 8aa29c0fe..d33a10c01 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -11,18 +11,17 @@ ************************************************************************* ** ** This app's single purpose is to emit parts of the Makefile code for -** building sqlite3's WASM build. The main motivation is to generate -** code which "can" be created via GNU Make's eval command but is +** sqlite3's canonical WASM build. The main motivation is to generate +** code which "could" be created via GNU Make's eval command but is ** highly illegible when constructed that way. Attempts to write this -** app in Bash and TCL have suffered from the problem that both -** require escaping $ symbols, making the resulting script code as -** illegible as the eval spaghetti we want to get away from. Writing -** it in C is, somewhat surprisingly, _slightly_ less illegible than -** writing it in bash, tcl, or native Make code. +** app in Bash and TCL have suffered from the problem that those +** languages require escaping $ symbols, making the resulting script +** code as illegible as the eval spaghetti we want to get away +** from. Maintaining it in C is, somewhat surprisingly, _slightly_ +** less illegible than writing it in bash, tcl, or native Make code. ** ** The emitted makefile code is not standalone - it depends on ** variables and $(call)able functions from the main makefile. -** */ #undef NDEBUG @@ -33,38 +32,145 @@ #define pf printf #define ps puts -/* Very common printf() args combo. */ -#define zNM zName, zMode /* -** Valid names for the zName arguments. +** Valid build names. Each build is a combination of one of these and +** one of JS_BUILD_MODES, but only certain combinations are legal. +** This macro and JS_BUILD_MODES exist solely for documentation +** purposes: they are not expanded into code anywhere. */ #define JS_BUILD_NAMES sqlite3 sqlite3-wasmfs /* -** Valid names for the zMode arguments of the "sqlite3" build. For the -** "sqlite3-wasmfs" build, only "esm" (ES6 Module) is legal. +** Valid build modes. For the "sqlite3-wasmfs" build, only "esm" (ES6 +** Module) is legal. */ #define JS_BUILD_MODES vanilla esm bundler-friendly node -/* Separator to help eyeballs find the different sections */ + +/* Separator to help eyeballs find the different output sections */ static const char * zBanner = "\n########################################################################\n"; /* +** Flags for use with BuildDef::flags and the 3rd argument to +** mk_pre_post(). +** +** Maintenance reminder: do not combine flags within this enum, +** e.g. LIBMODE_BUNDLER_FRIENDLY=0x02|LIBMODE_ESM, as that will lead +** to breakage in some of the flag checks. +*/ +enum LibModeFlags { + /* Indicates an ESM module build. */ + LIBMODE_ESM = 0x01, + /* Indicates a "bundler-friendly" build mode. */ + LIBMODE_BUNDLER_FRIENDLY = 0x02, + /* Indicates that this build is unsupported. Such builds are not + ** added to the 'all' target. The unsupported builds exist primarily + ** for experimentation's sake. */ + LIBMODE_UNSUPPORTED = 0x04, + /* Indicates a node.js-for-node.js build (untested and + ** unsupported). */ + LIBMODE_NODEJS = 0x08, + /* Indicates a wasmfs build (untested and unsupported). */ + LIBMODE_WASMFS = 0x10 +}; + +/* +** Info needed for building one combination of JS_BUILD_NAMES and +** JS_BUILD_MODE, noting that only a subset of those combinations are +** legal/sensical. +*/ +struct BuildDef { + const char *zName; /* Name from JS_BUILD_NAMES */ + const char *zMode; /* Name from JS_BUILD_MODES */ + int flags; /* Flags from LibModeFlags */ + const char *zJsOut; /* Name of generated sqlite3.js/.mjs */ + /* TODO: dynamically determine zJsOut based on zName, zMode, and + flags. */ + const char *zCmppD; /* Extra -D... flags for c-pp */ + const char *zEmcc; /* Extra flags for emcc */ +}; +typedef struct BuildDef BuildDef; + +/* +** The set of WASM builds for the library (as opposed to the apps +** (fiddle, speedtest1)). This array must end with an empty sentinel +** entry. Their order is mostly insignificant, but some makefile vars +** used by some builds are set up by prior builds. Because of that, +** the (sqlite3, vanilla), (sqlite3, esm), and (sqlite3, +** bundler-friendly) builds should be defined first (in that order). +*/ +const BuildDef aBuildDefs[] = { + {/* Core build */ + "sqlite3", "vanilla", 0, "$(sqlite3.js)", 0, 0}, + + {/* Core ESM */ + "sqlite3", "esm", LIBMODE_ESM, "$(sqlite3.mjs)", + "-Dtarget=es6-module", 0}, + + {/* Core bundler-friendly build. Untested and "not really" + ** supported, but required by the downstream npm subproject. + ** Testing these would require special-purpose node-based tools and + ** custom test apps. Or we can pass them off as-is to the npm + ** subproject and they spot failures pretty quickly ;). */ + "sqlite3", "bundler-friendly", + LIBMODE_BUNDLER_FRIENDLY | LIBMODE_ESM, + "$(dir.dout)/sqlite3-bundler-friendly.mjs", + "$(c-pp.D.sqlite3-esm) -Dtarget=es6-bundler-friendly", 0}, + + {/* node.js mode. Untested and unsupported. */ + "sqlite3", "node", LIBMODE_UNSUPPORTED | LIBMODE_NODEJS, + "$(dir.dout)/sqlite3-node.mjs", + "$(c-pp.D.sqlite3-bundler-friendly) -Dtarget=node", 0}, + + {/* Wasmfs build. Fully unsupported and largely untested. */ + "sqlite3-wasmfs", "esm" , + LIBMODE_UNSUPPORTED | LIBMODE_WASMFS | LIBMODE_ESM, + "$(dir.wasmfs)/sqlite3-wasmfs.mjs", + "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs", + "-sEXPORT_ES6 -sUSE_ES6_IMPORT_META"}, + + {/*End-of-list sentinel*/0,0,0,0,0,0} +}; + +/* ** Emits common vars needed by the rest of the emitted code (but not ** needed by makefile code outside of these generated pieces). */ static void mk_prologue(void){ + /* A 0-terminated list of makefile vars which we expect to have been + ** set up by this point in the build process. */ + char const * aRequiredVars[] = { + "dir.top", + "dir.api", "dir.dout", "dir.tmp", + "sqlite3-license-version.js", + "MAKEFILE", "MAKEFILE_LIST", + /* Fiddle... */ + "dir.fiddle", "dir.fiddle-debug", + "MAKEFILE.fiddle", + "EXPORTED_FUNCTIONS.fiddle", + /*"just-testing",*/ + 0 + }; + char const * zVar; + int i; + pf("%s# Build setup sanity checks...\n", zBanner); + for( i = 0; (zVar = aRequiredVars[i]); ++i ){ + pf("ifeq (,$(%s))\n", zVar); + pf(" $(error build process error: expecting make var $$(%s) to " + "have been set up by now)\n", zVar); + ps("endif"); + } pf("%s", zBanner); ps("# extern-post-js* and extern-pre-js* are files for use with"); ps("# Emscripten's --extern-pre-js and --extern-post-js flags."); - ps("extern-pre-js.js := $(dir.api)/extern-pre-js.js"); - ps("extern-post-js.js.in := $(dir.api)/extern-post-js.c-pp.js"); + ps("extern-pre-js.js = $(dir.api)/extern-pre-js.js"); + ps("extern-post-js.js.in = $(dir.api)/extern-post-js.c-pp.js"); ps("# Emscripten flags for --[extern-][pre|post]-js=... for the"); ps("# various builds."); - ps("pre-post-common.flags := --extern-pre-js=$(sqlite3-license-version.js)"); - ps("# pre-post-jses.deps.* = a list of dependencies for the"); - ps("# --[extern-][pre/post]-js files."); - ps("pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js)"); + ps("pre-post-common.flags = --extern-pre-js=$(sqlite3-license-version.js)"); + ps("# pre-post-jses.deps.* = a list of dependencies for the\n" + "# --[extern-][pre/post]-js files."); + ps("pre-post-jses.deps.common = $(extern-pre-js.js) $(sqlite3-license-version.js)"); { /* SQLITE.CALL.WASM-OPT = shell code to run $(1) (source wasm file @@ -127,7 +233,7 @@ static void mk_prologue(void){ "\t\techo -n 'After wasm-opt: '; \\\n" "\t\tls -l $(1); \\\n" "\telse \\\n" - "\t\techo 'WARNING: ignoring wasm-opt failure'; \\\n" + "\t\techo 'WARNING: ignoring wasm-opt failure for $(1)'; \\\n" "\tfi\n", zOptFlags ); @@ -137,52 +243,32 @@ static void mk_prologue(void){ } /* -** Flags for use with the 3rd argument to mk_pre_post() and -** mk_lib_mode(). -** -** Maintenance reminder: do not combine flags within this enum, -** e.g. LIBMODE_BUNDLER_FRIENDLY=0x02|LIBMODE_ESM, as that will lead -** to breakage in some of the flag checks. -*/ -enum LibModeFlags { - /* Indicates an ESM module build. */ - LIBMODE_ESM = 0x01, - /* Indicates a "bundler-friendly" build mode. */ - LIBMODE_BUNDLER_FRIENDLY = 0x02, - /* Indicates to _not_ add this build to the 'all' target. */ - LIBMODE_DONT_ADD_TO_ALL = 0x04, - /* Indicates a node.js-for-node.js build (untested and - ** unsupported). */ - LIBMODE_NODEJS = 0x08, - /* Indicates a wasmfs build (untested and unsupported). */ - LIBMODE_WASMFS = 0x10 -}; - -/* ** Emits makefile code for setting up values for the --pre-js=FILE, ** --post-js=FILE, and --extern-post-js=FILE emcc flags, as well as ** populating those files. */ static void mk_pre_post(const char *zName /* build name */, const char *zMode /* build mode */, - int flags /* LIBMODE_... mask */, const char *zCmppD /* optional -D flags for c-pp for the ** --pre/--post-js files. */){ +/* Very common printf() args combo. */ +#define zNM zName, zMode + pf("%s# Begin --pre/--post flags for %s-%s\n", zBanner, zNM); - pf("c-pp.D.%s-%s := %s\n", zNM, zCmppD ? zCmppD : ""); + pf("c-pp.D.%s-%s = %s\n", zNM, zCmppD ? zCmppD : ""); pf("pre-post-%s-%s.flags ?=\n", zNM); /* --pre-js=... */ - pf("pre-js.js.%s-%s := $(dir.tmp)/pre-js.%s-%s.js\n", + pf("pre-js.js.%s-%s = $(dir.tmp)/pre-js.%s-%s.js\n", zNM, zNM); - pf("$(pre-js.js.%s-%s): $(MAKEFILE_LIST)\n", zNM); + pf("$(pre-js.js.%s-%s): $(MAKEFILE_LIST) $(sqlite3-license-version.js)\n", zNM); #if 1 pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.%s-%s)," "$(c-pp.D.%s-%s)))\n", zNM, zNM); #else /* This part is needed if/when we re-enable the custom ** Module.instantiateModule() impl in api/pre-js.c-pp.js. */ - pf("pre-js.js.%s-%s.intermediary := $(dir.tmp)/pre-js.%s-%s.intermediary.js\n", + pf("pre-js.js.%s-%s.intermediary = $(dir.tmp)/pre-js.%s-%s.intermediary.js\n", zNM, zNM); pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.%s-%s.intermediary)," "$(c-pp.D.%s-%s) -Dcustom-Module.instantiateModule))\n", zNM, zNM); @@ -200,17 +286,17 @@ static void mk_pre_post(const char *zName /* build name */, #endif /* --post-js=... */ - pf("post-js.js.%s-%s := $(dir.tmp)/post-js.%s-%s.js\n", zNM, zNM); + pf("post-js.js.%s-%s = $(dir.tmp)/post-js.%s-%s.js\n", zNM, zNM); pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(post-js.js.in)," "$(post-js.js.%s-%s),$(c-pp.D.%s-%s)))\n", zNM, zNM); /* --extern-post-js=... */ - pf("extern-post-js.js.%s-%s := $(dir.tmp)/extern-post-js.%s-%s.js\n", zNM, zNM); + pf("extern-post-js.js.%s-%s = $(dir.tmp)/extern-post-js.%s-%s.js\n", zNM, zNM); pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.%s-%s)," "$(c-pp.D.%s-%s)))\n", zNM, zNM); /* Combined flags for use with emcc... */ - pf("pre-post-common.flags.%s-%s := " + pf("pre-post-common.flags.%s-%s = " "$(pre-post-common.flags) " "--post-js=$(post-js.js.%s-%s) " "--extern-post-js=$(extern-post-js.js.%s-%s)\n", zNM, zNM, zNM); @@ -219,30 +305,29 @@ static void mk_pre_post(const char *zName /* build name */, "--pre-js=$(pre-js.js.%s-%s)\n", zNM, zNM, zNM); /* Set up deps... */ - pf("pre-post-jses.%s-%s.deps := $(pre-post-jses.deps.common) " + pf("pre-post-jses.%s-%s.deps = $(pre-post-jses.deps.common) " "$(post-js.js.%s-%s) $(extern-post-js.js.%s-%s)\n", zNM, zNM, zNM); - pf("pre-post-%s-%s.deps := $(pre-post-jses.%s-%s.deps) $(dir.tmp)/pre-js.%s-%s.js\n", + pf("pre-post-%s-%s.deps = $(pre-post-jses.%s-%s.deps) $(dir.tmp)/pre-js.%s-%s.js\n", zNM, zNM, zNM); pf("# End --pre/--post flags for %s-%s%s", zNM, zBanner); +#undef zNM } /* ** Emits rules for the fiddle builds. -** */ -static void mk_fiddle(){ +static void mk_fiddle(void){ int i = 0; - mk_pre_post("fiddle-module","vanilla", 0, 0); + mk_pre_post("fiddle-module","vanilla", 0); for( ; i < 2; ++i ){ + /* 0==normal, 1==debug */ const char *zTail = i ? ".debug" : ""; const char *zDir = i ? "$(dir.fiddle-debug)" : "$(dir.fiddle)"; pf("%s# Begin fiddle%s\n", zBanner, zTail); - pf("fiddle-module.js%s := %s/fiddle-module.js\n", zTail, zDir); - pf("fiddle-module.wasm%s := " - "$(subst .js,.wasm,$(fiddle-module.js%s))\n", zTail, zTail); + pf("fiddle-module.js%s = %s/fiddle-module.js\n", zTail, zDir); pf("$(fiddle-module.js%s):%s $(MAKEFILE_LIST) $(MAKEFILE.fiddle) " "$(EXPORTED_FUNCTIONS.fiddle) " "$(fiddle.cses) $(pre-post-fiddle-module-vanilla.deps) " @@ -254,7 +339,9 @@ static void mk_fiddle(){ pf("\t$(bin.emcc) -o $@ $(fiddle.emcc-flags%s) " "$(pre-post-fiddle-module-vanilla.flags) $(fiddle.cses)\n", zTail); - pf("\t$(maybe-wasm-strip) $(fiddle-module.wasm%s)\n", zTail); + ps("\t@chmod -x $(basename $@).wasm"); + ps("\t@$(maybe-wasm-strip) $(basename $@).wasm"); + ps("\t@$(SQLITE.strip-createExportWrapper)"); pf("\t@cp -p $(SOAP.js) $(dir $@)\n"); if( 1==i ){/*fiddle.debug*/ pf("\tcp -p $(dir.fiddle)/index.html " @@ -263,13 +350,13 @@ static void mk_fiddle(){ "$(dir $@)\n"); } pf("\t@for i in %s/*.*js %s/*.html %s/*.wasm; do \\\n" - "\t\ttest -f $${i} || continue; \\\n" + "\t\ttest -f $${i} || continue; \\\n" "\t\tgzip < $${i} > $${i}.gz; \\\n" "\tdone\n", zDir, zDir, zDir); if( 0==i ){ ps("fiddle: $(fiddle-module.js)"); }else{ - ps("fiddle-debug: $(fiddle-module-debug.js)"); + ps("fiddle-debug: $(fiddle-module.js.debug)"); } pf("# End fiddle%s%s", zTail, zBanner); } @@ -280,138 +367,110 @@ static void mk_fiddle(){ ** by the combination of zName and zMode, each of which must be values ** from JS_BUILD_NAMES resp. JS_BUILD_MODES. */ -static void mk_lib_mode(const char *zName /* build name */, - const char *zMode /* build mode */, - int flags /* LIBMODE_... mask */, - const char *zApiJsOut /* name of generated sqlite3-api.js/.mjs */, - const char *zJsOut /* name of generated sqlite3.js/.mjs */, - const char *zCmppD /* extra -D flags for c-pp */, - const char *zEmcc /* extra flags for emcc */){ +static void mk_lib_mode(const BuildDef * pB){ const char * zWasmOut = "$(basename $@).wasm" /* The various targets named X.js or X.mjs (zJsOut) also generate ** X.wasm, and we need that part of the name to perform some ** post-processing after Emscripten generates X.wasm. */; - assert( zName ); - assert( zMode ); - assert( zApiJsOut ); - assert( zJsOut ); - if( !zCmppD ) zCmppD = ""; - if( !zEmcc ) zEmcc = ""; + assert( pB->zName ); + assert( pB->zMode ); + assert( pB->zJsOut ); +/* Very common printf() args combo. */ +#define zNM pB->zName, pB->zMode - pf("%s# Begin build [%s-%s]\n", zBanner, zNM); - pf("# zApiJsOut=%s\n# zJsOut=%s\n# zCmppD=%s\n", zApiJsOut, zJsOut, zCmppD); - pf("$(info Setting up build [%s-%s]: %s)\n", zNM, zJsOut); - mk_pre_post(zNM, flags, zCmppD); + pf("%s# Begin build [%s-%s]. flags=0x%02x\n", zBanner, zNM, pB->flags); + pf("# zJsOut=%s\n# zCmppD=%s\n", pB->zJsOut, + pB->zCmppD ? pB->zCmppD : "<none>"); + pf("$(info Setting up build [%s-%s]: %s)\n", zNM, pB->zJsOut); + mk_pre_post(zNM, pB->zCmppD); pf("\nemcc.flags.%s.%s ?=\n", zNM); - if( zEmcc[0] ){ - pf("emcc.flags.%s.%s += %s\n", zNM, zEmcc); + if( pB->zEmcc && pB->zEmcc[0] ){ + pf("emcc.flags.%s.%s += %s\n", zNM, pB->zEmcc); } - pf("$(eval $(call SQLITE.CALL.C-PP.FILTER, $(sqlite3-api.js.in), %s, %s))\n", - zApiJsOut, zCmppD); - /* target zJsOut */ - pf("%s: %s $(MAKEFILE_LIST) $(sqlite3-wasm.cfiles) $(EXPORTED_FUNCTIONS.api) " + /* target pB->zJsOut */ + pf("%s: $(MAKEFILE_LIST) $(sqlite3-wasm.cfiles) $(EXPORTED_FUNCTIONS.api) " "$(pre-post-%s-%s.deps) " "$(sqlite3-api.ext.jses)" /* ^^^ maintenance reminder: we set these as deps so that they - get copied into place early. That allows the developer to - reload the base-most test pages while the later-stage builds - are still compiling, which is especially helpful when running - builds with long build times (like -Oz). */ + ** get copied into place early. That allows the developer to + ** reload the base-most test pages while the later-stage builds + ** are still compiling, which is especially helpful when running + ** builds with long build times (like -Oz). */ "\n", - zJsOut, zApiJsOut, zNM); + pB->zJsOut, zNM); pf("\t@echo \"Building $@ ...\"\n"); + if( LIBMODE_UNSUPPORTED & pB->flags ){ + ps("\t@echo 'ACHTUNG: $@ is an unsupported build. " + "Use at your own risk.'"); + } pf("\t$(bin.emcc) -o $@ $(emcc_opt_full) $(emcc.flags) \\\n"); - pf("\t\t$(emcc.jsflags) -sENVIRONMENT=$(emcc.environment.%s) \\\n", zMode); + pf("\t\t$(emcc.jsflags) -sENVIRONMENT=$(emcc.environment.%s) \\\n", + pB->zMode); pf("\t\t$(pre-post-%s-%s.flags) \\\n", zNM); - pf("\t\t$(emcc.flags.%s) $(emcc.flags.%s.%s) \\\n", zName, zNM); + pf("\t\t$(emcc.flags.%s) $(emcc.flags.%s.%s) \\\n", pB->zName, zNM); pf("\t\t$(cflags.common) $(SQLITE_OPT) \\\n" "\t\t$(cflags.%s) $(cflags.%s.%s) \\\n" - "\t\t$(cflags.wasm_extra_init) $(sqlite3-wasm.cfiles)\n", zName, zNM); - if( (LIBMODE_ESM & flags) || (LIBMODE_NODEJS & flags) ){ + "\t\t$(cflags.wasm_extra_init) $(sqlite3-wasm.cfiles)\n", pB->zName, zNM); + if( (LIBMODE_ESM & pB->flags) || (LIBMODE_NODEJS & pB->flags) ){ /* TODO? Replace this $(call) with the corresponding makefile ** code. OTOH, we also use this $(call) in the speedtest1-wasmfs ** build, which is not part of the rules emitted by this ** program. */ pf("\t@$(call SQLITE.CALL.xJS.ESM-EXPORT-DEFAULT,1,%d)\n", - (LIBMODE_WASMFS & flags) ? 1 : 0); + (LIBMODE_WASMFS & pB->flags) ? 1 : 0); } - pf("\t@chmod -x %s; \\\n" - "\t\t$(maybe-wasm-strip) %s;\n", - zWasmOut, zWasmOut); + pf("\t@chmod -x %s\n", zWasmOut); + pf("\t@$(maybe-wasm-strip) %s\n", zWasmOut); pf("\t@$(call SQLITE.CALL.WASM-OPT,%s)\n", zWasmOut); - pf("\t@sed -i -e '/^var _sqlite3.*createExportWrapper/d' %s || exit; \\\n" - /* ^^^^^^ reminder: Mac/BSD sed has no -i flag */ - "\t\techo 'Stripped out createExportWrapper() parts.'\n", - zJsOut) /* Our JS code installs bindings of each WASM export. The - generated Emscripten JS file does the same using its - own framework, but we don't use those results and can - speed up lib init, and reduce memory cost - considerably, by stripping them out. */; + ps("\t@$(SQLITE.strip-createExportWrapper)"); /* - ** The above $(bin.emcc) call will write zJsOut and will create a - ** like-named .wasm file (zWasmOut). That .wasm file name gets - ** hard-coded into zJsOut so we need to, for some cases, patch - ** zJsOut to use the name sqlite3.wasm instead. Note that the + ** The above $(bin.emcc) call will write pB->zJsOut, a.k.a. $@, and + ** will create a like-named .wasm file (zWasmOut). That .wasm file + ** name gets hard-coded into $@ so we need to, for some cases, patch + ** zJsOut to use the name sqlite3.wasm instead. Note that the ** resulting .wasm file is identical for all builds for which zEmcc ** is empty. */ - if( (LIBMODE_BUNDLER_FRIENDLY & flags) - || (LIBMODE_NODEJS & flags) ){ - pf("\t@echo 'Patching $@ for %s.wasm...'; \\\n", zName); + if( (LIBMODE_BUNDLER_FRIENDLY & pB->flags) ){ + pf("\t@echo 'Patching $@ for %s.wasm...'; \\\n", pB->zName); pf("\t\trm -f %s; \\\n", zWasmOut); pf("\t\tsed -i -e 's/%s-%s.wasm/%s.wasm/g' $@ || exit;\n", - /* ^^^^^^ reminder: Mac/BSD sed has no -i flag */ - zNM, zName); + /* ^^^^^^ reminder: Mac/BSD sed has no -i flag but this + ** build process explicitly requires a Linux system. */ + zNM, pB->zName); pf("\t@ls -la $@\n"); - if( LIBMODE_BUNDLER_FRIENDLY & flags ){ + if( LIBMODE_BUNDLER_FRIENDLY & pB->flags ){ /* Avoid a 3rd occurrence of the bug fixed by 65798c09a00662a3, ** which was (in two cases) caused by makefile refactoring and ** not recognized until after a release was made with the broken - ** sqlite3-bundler-friendly.mjs: */ + ** sqlite3-bundler-friendly.mjs (which is used by the npm + ** subproject but is otherwise untested/unsupported): */ pf("\t@if grep -e '^ *importScripts(' $@; " "then echo 'ERROR: bug fixed in 65798c09a00662a3 has re-appeared'; " "exit 1; fi;\n"); } - }else{ pf("\t@ls -la %s $@\n", zWasmOut); } - if( 0==(LIBMODE_DONT_ADD_TO_ALL & flags) ){ - pf("all: %s\n", zJsOut); + if( 0==(LIBMODE_UNSUPPORTED & pB->flags) ){ + pf("all: %s\n", pB->zJsOut); } pf("# End build [%s-%s]%s", zNM, zBanner); +#undef zNM } int main(void){ int rc = 0; + const BuildDef *pB = &aBuildDefs[0]; pf("# What follows was GENERATED by %s. Edit at your own risk.\n", __FILE__); mk_prologue(); - mk_lib_mode("sqlite3", "vanilla", 0, - "$(sqlite3-api.js)", "$(sqlite3.js)", 0, 0); - mk_lib_mode("sqlite3", "esm", LIBMODE_ESM, - "$(sqlite3-api.mjs)", "$(sqlite3.mjs)", - "-Dtarget=es6-module", 0); - mk_lib_mode("sqlite3", "bundler-friendly", - LIBMODE_BUNDLER_FRIENDLY | LIBMODE_ESM, - "$(sqlite3-api-bundler-friendly.mjs)", - "$(sqlite3-bundler-friendly.mjs)", - "$(c-pp.D.sqlite3-esm) -Dtarget=es6-bundler-friendly", 0); - mk_lib_mode("sqlite3" , "node", - LIBMODE_NODEJS | LIBMODE_DONT_ADD_TO_ALL, - "$(sqlite3-api-node.mjs)", "$(sqlite3-node.mjs)", - "$(c-pp.D.sqlite3-bundler-friendly) -Dtarget=node", 0); - mk_lib_mode("sqlite3-wasmfs", "esm" , - LIBMODE_WASMFS | LIBMODE_ESM | LIBMODE_DONT_ADD_TO_ALL, - /* The sqlite3-wasmfs build is optional and needs to be invoked - ** conditionally using info we don't have here. */ - "$(sqlite3-api-wasmfs.mjs)", "$(sqlite3-wasmfs.mjs)", - "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs", - "-sEXPORT_ES6 -sUSE_ES6_IMPORT_META"); - + for( ; pB->zName; ++pB ){ + mk_lib_mode( pB ); + } mk_fiddle(); - mk_pre_post("speedtest1","vanilla", 0, 0); - mk_pre_post("speedtest1-wasmfs","esm", 0, + mk_pre_post("speedtest1","vanilla", 0); + mk_pre_post("speedtest1-wasmfs","esm", "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs"); return rc; } diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index 8c9a77dc5..ba0d22fb6 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -279,7 +279,7 @@ opt.innerHTML = lbl; opt.value = f; if(preselectedFlags.indexOf(f) >= 0) opt.selected = true; - }); + }); const cbReverseLog = E('#cb-reverse-log-order'); const lblReverseLog = E('#lbl-reverse-log-order'); if(cbReverseLog.checked){ diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html index 9cc20924e..a841c7fa0 100644 --- a/ext/wasm/speedtest1.html +++ b/ext/wasm/speedtest1.html @@ -23,11 +23,10 @@ </figure> <div class="emscripten" id="module-status">Downloading...</div> <div class="emscripten"> - <progress value="0" max="100" id="module-progress" hidden='1'></progress> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> </div><!-- /emscripten bits --> <div class='warning'>This page starts running the main exe when it loads, which will - block the UI until it finishes! Adding UI controls to manually configure and start it - are TODO.</div> + block the UI until it finishes!</div> </div> <div class='warning'>Achtung: running it with the dev tools open may <em>drastically</em> slow it down. For faster results, keep the dev @@ -118,7 +117,7 @@ argv.push("--vfs", vfs); log2('',"Using VFS:",vfs); if('kvvfs' === vfs){ - forceSize = 4 /* 5 uses approx. 4.96mb */; + forceSize = 2 /* >2 is too big as of mid-2025 */; dbFile = 'session'; log2('warning',"kvvfs VFS: forcing --size",forceSize, "and filename '"+dbFile+"'."); diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index d30e59e38..dd70024ab 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -41,7 +41,7 @@ ES6 worker module build: - ./c-pp -f tester1.c-pp.js -o tester1-esm.js -Dtarget=es6-module + ./c-pp -f tester1.c-pp.js -o tester1-esm.mjs -Dtarget=es6-module */ //#if target=es6-module import {default as sqlite3InitModule} from './jswasm/sqlite3.mjs'; @@ -221,7 +221,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; else if(filter instanceof Function) pass = filter(err); else if('string' === typeof filter) pass = (err.message === filter); if(!pass){ - throw new Error(msg || ("Filter rejected this exception: "+err.message)); + throw new Error(msg || ("Filter rejected this exception: <<"+err.message+">>")); } return this; }, @@ -1209,6 +1209,104 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } }) + + //////////////////////////////////////////////////////////////////// + .t({ + name: "oo1.DB/Stmt.wrapDbHandle()", + test: function(sqlite3){ + /* Maintenance reminder: this function is early in the list to + demonstrate that the wrappers for this.db created by this + function do not interfere with downstream tests, e.g. by + closing this.db.pointer. */ + //sqlite3.config.debug("Proxying",this.db); + const misuseMsg = "SQLITE_MISUSE: Argument must be a WASM sqlite3 pointer"; + T.mustThrowMatching(()=>sqlite3.oo1.DB.wrapHandle(this.db), misuseMsg) + .mustThrowMatching(()=>sqlite3.oo1.DB.wrapHandle(0), misuseMsg); + let dw = sqlite3.oo1.DB.wrapHandle(this.db.pointer); + //sqlite3.config.debug('dw',dw); + T.assert( dw, '!!dw' ) + .assert( dw instanceof sqlite3.oo1.DB, 'dw is-a oo1.DB' ) + .assert( dw.pointer, 'dw.pointer' ) + .assert( dw.pointer === this.db.pointer, 'dw.pointer===db.pointer' ) + .assert( dw.filename === this.db.filename, 'dw.filename===db.filename' ); + + T.assert( dw === dw.exec("select 1") ); + let q; + try { + q = dw.prepare("select 1"); + T.assert( q.step() ) + .assert( !q.step() ); + }finally{ + if( q ) q.finalize(); + } + dw.close(); + T.assert( !dw.pointer ) + .assert( this.db === this.db.exec("select 1") ); + dw = undefined; + + let pDb = 0, pStmt = 0; + const stack = wasm.pstack.pointer; + try { + const ppOut = wasm.pstack.allocPtr(); + T.assert( 0===wasm.peekPtr(ppOut) ); + let rc = capi.sqlite3_open_v2( ":memory:", ppOut, + capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE, + 0); + T.assert( 0===rc, 'open_v2()' ); + pDb = wasm.peekPtr(ppOut); + wasm.pokePtr(ppOut, 0); + T.assert( pDb>0, 'pDb>0' ); + const pTmp = pDb; + dw = sqlite3.oo1.DB.wrapHandle(pDb, true); + pDb = 0; + //sqlite3.config.debug("dw",dw); + T.assert( pTmp===dw.pointer, 'pDb===dw.pointer' ); + T.assert( dw.filename === "", "dw.filename == "+dw.filename ); + let q = dw.prepare("select 1"); + try { + T.assert( q.step(), "step()" ); + T.assert( !q.step(), "!step()" ); + }finally{ + q.finalize(); + q = undefined; + } + T.assert( dw===dw.exec("select 1") ); + dw.affirmOpen(); + const select1 = "select 1"; + rc = capi.sqlite3_prepare_v2( dw, select1, -1, ppOut, 0 ); + T.assert( 0===rc, 'prepare_v2() rc='+rc ); + pStmt = wasm.peekPtr(ppOut); + T.assert( pStmt && wasm.isPtr(pStmt), 'pStmt is valid?' ); + try { + //log( "capi.sqlite3_sql() =",capi.sqlite3_sql(pStmt)); + T.assert( select1 === capi.sqlite3_sql(pStmt), 'SQL mismatch' ); + q = sqlite3.oo1.Stmt.wrapHandle(dw, pStmt, false); + //log("q@"+pStmt+" does not own handle"); + T.assert( q.step(), "step()" ) + .assert( !q.step(), "!step()" ); + q.finalize(); + q = undefined; + T.assert( select1 === capi.sqlite3_sql(pStmt), 'SQL mismatch' + /* This will fail if we've mismanaged pStmt's lifetime */); + q = sqlite3.oo1.Stmt.wrapHandle(dw, pStmt, true); + pStmt = 0; + q.reset(); + T.assert( q.step(), "step()" ) + .assert( !q.step(), "!step()" ); + }finally{ + if( pStmt ) capi.sqlite3_finalize(pStmt) + if( q ) q.finalize(); + } + + }finally{ + wasm.pstack.restore(stack); + if( pDb ){ capi.sqlite3_close_v2(pDb); } + else if( dw ){ dw.close(); } + } + } + })/*oo1.DB/Stmt.wrapHandle()*/ + //////////////////////////////////////////////////////////////////// .t('sqlite3_db_config() and sqlite3_db_status()', function(sqlite3){ let rc = capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, 0, 0); @@ -1263,12 +1361,12 @@ globalThis.sqlite3InitModule = sqlite3InitModule; capi.sqlite3_stmt_status( st, capi.SQLITE_STMTSTATUS_RUN, 0 ) === 0) - .assert(!st._mayGet) .assert('a' === st.getColumnName(0)) .mustThrowMatching(()=>st.columnCount=2, /columnCount property is read-only/) .assert(1===st.columnCount) .assert(0===st.parameterCount) + .assert(0===capi.sqlite3_bind_parameter_count(st)) .mustThrow(()=>st.bind(1,null)) .assert(true===st.step()) .assert(3 === st.get(0)) @@ -1287,9 +1385,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(1===st.get(0,capi.SQLITE_BLOB).length) .assert(st.getBlob(0) instanceof Uint8Array) .assert('3'.charCodeAt(0) === st.getBlob(0)[0]) - .assert(st._mayGet) .assert(false===st.step()) - .assert(!st._mayGet) + .mustThrowMatching(()=>st.get(0), + "Stmt.step() has not (recently) returned true.") .assert( capi.sqlite3_stmt_status( st, capi.SQLITE_STMTSTATUS_RUN, 0 @@ -1297,11 +1395,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(this.progressHandlerCount>0 || wasm.compileOptionUsed('OMIT_PROGRESS_CALLBACK'), - "Expecting progress callback."). - assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). - assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). - assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). - assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); + "Expecting progress callback."); }finally{ rc = st.finalize(); } @@ -1350,7 +1444,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(pVfsDb > 0) .assert(pVfsMem !== pVfsDflt /* memdb lives on top of the default vfs */) - .assert(pVfsDb === pVfsDflt || pVfsdb === pVfsMem) + .assert(pVfsDb === pVfsDflt || pVfsDb === pVfsMem) ; /*const vMem = new capi.sqlite3_vfs(pVfsMem), vDflt = new capi.sqlite3_vfs(pVfsDflt), @@ -1495,6 +1589,8 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let st = db.prepare("update t set b=:b where a='blob'"); try { T.assert(0===st.columnCount) + .assert(1===st.parameterCount) + .assert(1===capi.sqlite3_bind_parameter_count(st)) .assert( false===st.isReadOnly() ); const ndx = st.getParamIndex(':b'); T.assert(1===ndx); @@ -2193,7 +2289,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; "back into JS because of the lack of 64-bit integer support."); } }finally{ - const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); + //const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); //log("x=",x,"y=",y,"z=",z); // just looking at the alignment w.scopedAllocPop(stack); } @@ -2673,50 +2769,70 @@ globalThis.sqlite3InitModule = sqlite3InitModule; || "Only available in main thread."), test: function(sqlite3){ this.kvvfsUnlink(); - let db; - const encOpt1 = 1 - ? {textkey: 'foo'} - : {key: 'foo'}; - const encOpt2 = encOpt1.textkey - ? encOpt1 - : {hexkey: new Uint8Array([0x66,0x6f,0x6f]/*==>"foo"*/)} - try{ - db = new this.JDb({ - filename: this.kvvfsDbFile, - ...encOpt1 - }); - db.exec([ - "create table t(a,b);", - "insert into t(a,b) values(1,2),(3,4)" - ]); - db.close(); - let err; - try{ - db = new this.JDb({ - filename: this.kvvfsDbFile, - flags: 'ct' + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + const ctoropt = { + filename: this.kvvfsDbFile + //vfs: 'kvvfs' + //,flags: 'ct' + }; + try { + if (initDb) { + initDb = false; + db = new this.JDb({ + ...ctoropt, + [keyKey]: key + }); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + // Ensure that it's actually encrypted... + let err; + try { + db = new this.JDb(ctoropt); + T.assert(db, 'db opened') /* opening is fine, but... */; + db.exec("select 1 from sqlite_schema"); + console.warn("(should not be reached) sessionStorage =", sessionStorage); + } catch (e) { + err = e; + } finally { + db.close() + } + T.assert(err, "Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + //console.debug('tryKey()',arguments); + db = new sqlite3.oo1.DB({ + ...ctoropt, + vfs: 'kvvfs', + [keyKey]: key }); - T.assert(db) /* opening is fine, but... */; - db.exec("select 1 from sqlite_schema"); - console.warn("sessionStorage =",sessionStorage); - }catch(e){ - err = e; - }finally{ - db.close(); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); } - T.assert(err,"Expecting an exception") - .assert(sqlite3.capi.SQLITE_NOTADB==err.resultCode, - "Expecting NOTADB"); - db = new sqlite3.oo1.DB({ - filename: this.kvvfsDbFile, - vfs: 'kvvfs', - ...encOpt2 - }); - T.assert( 4===db.selectValue('select sum(a) from t') ); - }finally{ - if( db ) db.close(); - this.kvvfsUnlink(); - } + }.bind(this); + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + this.kvvfsUnlink(); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + this.kvvfsUnlink(); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); + this.kvvfsUnlink(); } })/*kvvfs with SEE*/ //#endif enable-see @@ -2836,6 +2952,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }, 9 ); + T.assert( 0==rc ); db.transaction((d)=>{ d.exec([ "create table t(a);", @@ -2849,8 +2966,10 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(2 === countHook[capi.SQLITE_UPDATE]) .assert(1 === countHook[capi.SQLITE_DELETE]); //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = true; - db.close(); + T.assert( !!capi.sqlite3_preupdate_hook(db, 0, 0) ); //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = false; + T.assert( !capi.sqlite3_preupdate_hook(db, 0, 0) ); + db.close(); } })/*pre-update hooks*/ ;/*end hook API tests*/ @@ -3051,7 +3170,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(6 === db.selectValue('select count(*) from p')). assert( this.opfsImportSize == exp.byteLength ); db.close(); - const unlink = this.opfsUnlink = + this.opfsUnlink = (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink("opfs",fn); this.opfsUnlink(filename); T.assert(!(await sqlite3.opfs.entryExists(filename))); @@ -3302,7 +3421,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(true === await u3.removeVfs()) .assert(false === await P3b.removeVfs()); } - }/*OPFS SAH Pool sanity checks*/) + }/*OPFS SAH Pool sanity checks*/); //////////////////////////////////////////////////////////////////////// T.g('Misc. APIs') @@ -3311,6 +3430,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; db.exec("create table t(a)"); const stmt = db.prepare("insert into t(a) values($a)"); T.assert( 1===capi.sqlite3_bind_parameter_count(stmt) ) + .assert( 1===stmt.parameterCount ) .assert( 1===capi.sqlite3_bind_parameter_index(stmt, "$a") ) .assert( 0===capi.sqlite3_bind_parameter_index(stmt, ":a") ) .assert( 1===stmt.getParamIndex("$a") ) @@ -3323,6 +3443,44 @@ globalThis.sqlite3InitModule = sqlite3InitModule; db.close(); }) + /** + Ensure that certain Stmt members throw when called + via DB.exec(). + */ + .t('locked-by-exec() APIs', function(sqlite3){ + const db = new sqlite3.oo1.DB(); + db.exec("create table t(a);insert into t(a) values(1);"); + let checkCount = 0; + const checkOp = function(op){ + ++checkCount; + T.mustThrowMatching(() => { + db.exec({ + sql: "select ?1", + bind: op, + callback: (row, stmt) => { + switch (row[0]) { + case 'bind': stmt.bind(1); break; + case 'finalize': + case 'clearBindings': + case 'reset': + case 'step': stmt[op](); break; + } + } + }); + }, /^Operation is illegal when statement is locked.*/) + }; + try{ + checkOp('bind'); + checkOp('finalize'); + checkOp('clearBindings'); + checkOp('reset'); + checkOp('step'); + T.assert(5===checkCount); + }finally{ + db.close(); + } + }) + //////////////////////////////////////////////////////////////////// .t("Misc. stmt_...", function(sqlite3){ const db = new sqlite3.oo1.DB(); @@ -3353,6 +3511,16 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert( 1===n ) .assert( 0===capi.sqlite3_stmt_busy(stmt) ) .assert( !stmt.isBusy() ); + + if( wasm.exports.sqlite3_column_origin_name ){ + log("Column metadata APIs enabled"); + T.assert( "t" === capi.sqlite3_column_table_name(stmt, 0)) + .assert("a" === capi.sqlite3_column_origin_name(stmt, 0)) + .assert("main" === capi.sqlite3_column_database_name(stmt, 0)) + }else{ + log("Column metadata APIs not enabled"); + } // column metadata APIs + stmt.finalize(); db.close(); }) @@ -3364,7 +3532,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; capi.sqlite3_interrupt(db); T.assert( 0!==capi.sqlite3_is_interrupted(db) ); db.close(); - }) + }); //////////////////////////////////////////////////////////////////////// T.g('Bug Reports') @@ -3384,10 +3552,10 @@ globalThis.sqlite3InitModule = sqlite3InitModule; sql: "SELECT * FROM f order by path", rowMode: 'array' }); - const dump = function(lbl){ + /*const dump = function(lbl){ let rc = fetchEm(); log((lbl ? (lbl+' results') : ''),rc); - }; + };*/ //dump('Full fts table'); let rc = fetchEm(); T.assert(3===rc.length); diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make index 2c6fa35bd..0d1fb4043 100644 --- a/ext/wasm/wasmfs.make +++ b/ext/wasm/wasmfs.make @@ -5,31 +5,28 @@ # sqlite3.wasm. It is intended to be "include"d from the main # GNUMakefile. ######################################################################## -MAKEFILE.wasmfs := $(lastword $(MAKEFILE_LIST)) +MAKEFILE.wasmfs = $(lastword $(MAKEFILE_LIST)) # ensure that the following message starts on line 10 or higher for proper # $(warning) alignment! ifneq (1,$(MAKING_CLEAN)) $(warning !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!) - $(warning !! The WASMFS build is not well-supported. WASMFS is a proverbial) - $(warning !! moving target, sometimes changing in incompatible ways between) - $(warning !! Emscripten versions. This build is provided for adventurous folks) - $(warning !! and is not a supported deliverable of the SQLite project.) + $(warning !! The WASMFS build is unsupported. Use at your own risk. $(warning !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!) endif -sqlite3-wasmfs.js := $(dir.wasmfs)/sqlite3-wasmfs.js -sqlite3-wasmfs.wasm := $(dir.wasmfs)/sqlite3-wasmfs.wasm +sqlite3-wasmfs.js = $(dir.wasmfs)/sqlite3-wasmfs.js +sqlite3-wasmfs.wasm = $(dir.wasmfs)/sqlite3-wasmfs.wasm ######################################################################## # emcc flags for .c/.o. -cflags.sqlite3-wasmfs := +cflags.sqlite3-wasmfs = cflags.sqlite3-wasmfs += -std=c99 -fPIC cflags.sqlite3-wasmfs += -pthread cflags.sqlite3-wasmfs += -DSQLITE_ENABLE_WASMFS ######################################################################## # emcc flags specific to building the final .js/.wasm file... -emcc.flags.sqlite3-wasmfs := +emcc.flags.sqlite3-wasmfs = emcc.flags.sqlite3-wasmfs += \ -sEXPORTED_RUNTIME_METHODS=wasmMemory # wasmMemory ==> for -sIMPORTED_MEMORY @@ -43,7 +40,7 @@ emcc.flags.sqlite3-wasmfs += -Wno-limited-postlink-optimizations emcc.flags.sqlite3-wasmfs += -sMEMORY64=0 emcc.flags.sqlite3-wasmfs += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) # ^^^^ 64MB is not enough for WASMFS/OPFS test runs using batch-runner.js -sqlite3-wasmfs.fsflags := -pthread -sWASMFS \ +sqlite3-wasmfs.fsflags = -pthread -sWASMFS \ -sPTHREAD_POOL_SIZE=1 \ -sERROR_ON_UNDEFINED_SYMBOLS=0 -sLLD_REPORT_UNDEFINED # ^^^^^ why undefined symbols are necessary for the wasmfs build is anyone's guess. @@ -53,10 +50,9 @@ emcc.flags.sqlite3-wasmfs += -sALLOW_MEMORY_GROWTH=0 # USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, # see https://github.com/WebAssembly/design/issues/1271 [-Wpthreads-mem-growth] # And, indeed, it runs slowly if memory is permitted to grow. -#emcc.flags.sqlite3-wasmfs.vanilla := -#emcc.flags.sqlite3-wasmfs.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META -all: $(sqlite3-wasmfs.mjs) -$(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(MAKEFILE.wasmfs) +#emcc.flags.sqlite3-wasmfs.vanilla = +#emcc.flags.sqlite3-wasmfs.esm = -sEXPORT_ES6 -sUSE_ES6_IMPORT_META +$(sqlite3-wasmfs.js) $(dir.wasmfs)/sqlite3-wasmfs.mjs: $(MAKEFILE.wasmfs) ######################################################################## # Build quirk: we cannot build BOTH .js and .mjs with our current # build infrastructure because the supplemental *.worker.js files get @@ -68,31 +64,31 @@ $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(MAKEFILE.wasmfs) # names is that it means that the corresponding .wasm file is also # built/saved multiple times. It is likely that anyone wanting to use # WASMFS will want an ES6 module, so that's what we build here. -wasmfs.build.ext := mjs -$(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(SOAP.js.bld) +wasmfs.build.ext = mjs +$(sqlite3-wasmfs.js) $(dir.wasmfs)/sqlite3-wasmfs.mjs: $(SOAP.js.bld) ifeq (js,$(wasmfs.build.ext)) $(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.js) wasmfs: $(sqlite3-wasmfs.js) else - $(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.mjs) - wasmfs: $(sqlite3-wasmfs.mjs) + $(sqlite3-wasmfs.wasm): $(dir.wasmfs)/sqlite3-wasmfs.mjs + wasmfs: $(dir.wasmfs)/sqlite3-wasmfs.mjs endif +all: wasmfs ######################################################################## # speedtest1 for wasmfs. -speedtest1-wasmfs.mjs := $(dir.wasmfs)/speedtest1-wasmfs.mjs -speedtest1-wasmfs.wasm := $(subst .mjs,.wasm,$(speedtest1-wasmfs.mjs)) -emcc.flags.speedtest1-wasmfs := $(sqlite3-wasmfs.fsflags) +speedtest1-wasmfs.mjs = $(dir.wasmfs)/speedtest1-wasmfs.mjs +speedtest1-wasmfs.wasm = $(subst .mjs,.wasm,$(speedtest1-wasmfs.mjs)) +emcc.flags.speedtest1-wasmfs = $(sqlite3-wasmfs.fsflags) emcc.flags.speedtest1-wasmfs += $(SQLITE_OPT) emcc.flags.speedtest1-wasmfs += -sALLOW_MEMORY_GROWTH=0 emcc.flags.speedtest1-wasmfs += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) -#$(eval $(call call-make-pre-js,speedtest1-wasmfs,ems)) +#$(info speedtest DEPS=pre-post-sqlite3-wasmfs-esm.deps=$(pre-post-sqlite3-wasmfs-esm.deps)) $(speedtest1-wasmfs.mjs): $(speedtest1.cfiles) $(sqlite3-wasmfs.js) \ - $(MAKEFILE) $(MAKEFILE.wasmfs) \ - $(pre-post-sqlite3-wasmfs-esm.deps) \ + $(MAKEFILE) $(MAKEFILE.wasmfs) $(pre-post-sqlite3-wasmfs-esm.deps) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." - $(emcc.bin) \ + $(bin.emcc) \ $(pre-post-sqlite3-wasmfs-esm.flags) \ $(cflags.common) \ $(cflags.sqlite3-wasmfs) \ @@ -2113,6 +2113,10 @@ tool-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl +snapshot-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ + sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl + strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) + ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl --snapshot clean-tool-zip: rm -f sqlite-tools-*.zip clean: clean-tool-zip @@ -1,12 +1,12 @@ -C Enforce\sjudgmental\styping\son\sSTORED\sgenerated\scolumns\sfor\sSTRICT\ntables.\s\s[forum:/forumpost/6caf195248a849e4|Forum\spost\s6caf195248]. -D 2025-06-18T16:17:00.665 +C Remove\sthe\s<i>experimental</i>\slsm1\sextension\sfrom\strunk,\sin\sas\smuch\sas\nreaders\swere\sthinking\sthat\sthis\swas\sa\ssupported\sextension\sand\swere\sreporting\nbugs\sagainst\sit. +D 2025-08-05T12:01:43.031 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d -F Makefile.in d8bc4aee9fb645c9f2ff0e3a30585d17a0df076bb6a33f0f20bab4999abb45a0 +F Makefile.in a6c14b6906f5322920dd5d1ff9a84808337911068f2a6e777ec7088a87cbc3a8 F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 -F Makefile.msc ec2011bbdfc917d6a1c7c173dabb29c14ead0dd8e2e0b67278a00ae4ba576a77 +F Makefile.msc e17608035fb75d3e5b322f846c6c3cbe80a5f34abf5df9e400c40dd55436dc14 F README.md e28077cfbef795e99c9c75ed95aa7257a1166709b562076441a8506ac421b7c1 F VERSION 16eddb43056a79c1977427ab7a05f3457c373fa159dcdced8754eb89ce7e06b8 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -26,7 +26,7 @@ F autoconf/tea/Makefile.in bf6b43eafcd18766d81a8f0085cfc9cb051d8abae9031a8e7c3f5 F autoconf/tea/README.txt 23475876343498ef2b514cc7510e8f1559a17e8e03fbc7a41c1c8a3b89e7b7e3 F autoconf/tea/_teaish.tester.tcl.in 8253b44be88e2e3f21de95a65d3a90c2be8e70b7bdd08a5b80e337ba7402f8f1 F autoconf/tea/auto.def ce95b9450e2fa4ba5dc857e208fe10f4e6f2d737796ac3278aee6079db417529 -F autoconf/tea/configure d0b12b984edca6030d1976375b80157ac78b5b90a5b4f0dcee39357f63f4a80b x +F autoconf/tea/configure 993eb27dafb35253965f9c0eb0eeefd113cae0508361c8fd90a4b58c3caf14ec x F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in e07da6b94561f4aa382bab65b1ccceb04701b97bf59d007c1d1f20a222b22d07 F autoconf/tea/teaish.tcl 81feb417e718ed75cdd7e2fdf6771f3da80dae97377a90c4d5b62b3754abbf1d @@ -46,8 +46,8 @@ F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F autosetup/jimsh0.c 563b966c137a4ce3c9333e5196723b7ac0919140a9d7989eb440463cd855c367 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 6aac1eb3059fc511c8e1659f33b96eb2a216a371ed9b28be6661374061be9b15 -F autosetup/sqlite-config.tcl 0bcb12c99a1673a2009164a3318890d5820701d329bf12702daa96a38d064bde +F autosetup/proj.tcl 7eaa83ccb6f5b250ee9192fa914918b0e2075edacf984c75fbf82cc2a6449c6c +F autosetup/sqlite-config.tcl 3177dedd7bd49465fa06677fd743c4966eee5702a9ddf4914c2c1af5e5972a52 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F autosetup/teaish/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca F autosetup/teaish/core.tcl aee092fc71986d1272b835ea7492bb55ffc213a289502e4f14da80cf67b7e3c3 @@ -113,8 +113,8 @@ F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 F ext/fts5/fts5_expr.c be9e5f7f11d87e7bd3680832c93c13050fe351994b5052b0215c2ef40312c23a F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 -F ext/fts5/fts5_index.c d171f2a507abccb3d524bf461b01f0d3971a9bf221be622ac7c671a991cb62ee -F ext/fts5/fts5_main.c 57933c18efe1058d8871199875c7a59744dabc3904f3aefbf9ff4a4e11fc79e2 +F ext/fts5/fts5_index.c 2a1be0fb3c1b185f84b08b8032ba332c82defa182ff125833c0fecba0a4938b0 +F ext/fts5/fts5_main.c e558225168845dc708abeb2ad10415696e5a3249bcba1810ba3c7ef80764962e F ext/fts5/fts5_storage.c 19bc7c4cbe1e6a2dd9849ef7d84b5ca1fcbf194cefc3e386b901e00e08bf05c2 F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 F ext/fts5/fts5_test_mi.c 4308d5658cb1f5eee5998dcbaac7d5bdf7a2ef43c8192ca6e0c843f856ccee26 @@ -126,7 +126,7 @@ F ext/fts5/fts5_vocab.c ff0441c4ea165081e8152dec6d29056faa0cdc281a9f218a00e3d7aa F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe -F ext/fts5/test/fts5aa.test 015c81b84d53bfcedd77d624202c8b02e9f0cbbb4b51688e3a9c9f90bccbb4ac +F ext/fts5/test/fts5aa.test cf4ff6180873bbc131666ba846ddd90148fcb61c20aad089711d3511cce24300 F ext/fts5/test/fts5ab.test 4bdb619fee409e11417e8827e320b857e42e926a01a0408fc9f143ec870a6ced F ext/fts5/test/fts5ac.test 4a73626de86f3d17c95738034880c4f0de8d54741fb943d819b528373657e59b F ext/fts5/test/fts5ad.test 058e616612964e61d19f70295f0e6eaedceb4b29b1fbf4f859615ef7e779dc22 @@ -160,14 +160,14 @@ F ext/fts5/test/fts5contentless2.test 70ffe6c611d8f278240da56734df8a77948f04e273 F ext/fts5/test/fts5contentless3.test 75eaae5ad6b284ee447788943974d323228f27cc35a1681da997135cff95bc6a F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b05ea89964c863833b6eff8 F ext/fts5/test/fts5contentless5.test 38cd0392c730dc7090c550321ce3c24ba4c392bc97308b51a4180e9959dca7b5 -F ext/fts5/test/fts5corrupt.test 6485f721b88ba355ca5d701e7ee87a4efa3ea578d8e6adb26f51ef956c8328bd -F ext/fts5/test/fts5corrupt2.test 335911e3f68b9625d850325f9e29a128db3f4276a8c9d4e32134580da8f924c4 -F ext/fts5/test/fts5corrupt3.test 3420ad30bf9e9bbdbd43b3224c582431744899530a65b11b60ddacdf14200e19 +F ext/fts5/test/fts5corrupt.test 237fce1c3261bb3a5bec333b0f0dbf5b105ec32627ef14cccbda3cfe13833193 +F ext/fts5/test/fts5corrupt2.test 4a03a158c2cb617c9f76d26b35c1ef2534124bc0bbddcea38dfd5b170ebea27b +F ext/fts5/test/fts5corrupt3.test 43d6a836892d79ab738ab89b3b6f4ae46c07ee966193e4b357bbb14e7f81d5da F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93f68f7d7a0d7d7dd733 -F ext/fts5/test/fts5corrupt5.test bcf0801b0c991eadae3cb8e978e82b4bf01412cb4df41874a90d5aa279c7cc96 +F ext/fts5/test/fts5corrupt5.test 73985d4fe6d8f0d5d5c7bcf79ae7c6522c376cd6ad710a0ff2f26e0c2e222abe F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 -F ext/fts5/test/fts5corrupt7.test 4e830875c33b9ea3c4cf1ba71e692b63893cbb4faae8c69b1071889dc26e211c -F ext/fts5/test/fts5corrupt8.test b81d802e41631e98100f49a1aadeeffef860e30a62d6ed7d743c2797c477239e +F ext/fts5/test/fts5corrupt7.test 814aab492d7a09abb5bfdd81cc66fc206d7f3868f9a3bae91876e02efc466fb3 +F ext/fts5/test/fts5corrupt8.test 0b10750caf8aa23fa1c379ca4caf6130d41454505e4d5315590f4061eedcbe44 F ext/fts5/test/fts5delete.test 2a5008f8b1174ef41d1974e606928c20e4f9da77d9f8347aed818994d89cced4 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 F ext/fts5/test/fts5determin.test 1b77879b2ae818b5b71c859e534ee334dac088b7cf3ff3bf76a2c82b1c788d11 @@ -197,11 +197,11 @@ F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e F ext/fts5/test/fts5hash.test fd3e0367fbf0b0944d6936fdb22696350f57b9871069c6766251578a103e8a14 -F ext/fts5/test/fts5integrity.test 646796671205dae46af5bb12a49b5696483cfe8e12d71d21454940b13ace95ab +F ext/fts5/test/fts5integrity.test c423ce16fd1ccadcac7fc22f794226b2bb00f5a187c0ab1d9f8502521b1bae05 F ext/fts5/test/fts5integrity2.test 4c3636615c0201232c44a8105d5cb14fd5499fd0ee3014d7ffd7e83aac76ece8 F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba1044c168db0e106f9ac92c F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 -F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad +F ext/fts5/test/fts5leftjoin.test 1c14b51f4d1344a89e488160882f05a2246dd7e70c5cf077c8fb473e03c66338 F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c F ext/fts5/test/fts5locale.test 83ba7ee12628b540d3098f39c39c1de0c0440eddff8f7512c8c698d0c4a3ae3c F ext/fts5/test/fts5matchinfo.test bc9e74157773db7f00aec1e85587f1145956ebdf1672c136f0f04323b2752aa0 @@ -229,7 +229,7 @@ F ext/fts5/test/fts5prefix.test c0b7842f1a2d830c0b146cd438a95ea4c5a25635719ed0d9 F ext/fts5/test/fts5prefix2.test a5bb43b8a2687efafa7ac4e5ccff6812015cf8cf18e3086bb0eb3126f30fbbf6 F ext/fts5/test/fts5query.test 0320a7a4b58a6e3e50ec8910b301649da90ace675001f9e0bf6392750ad4591d F ext/fts5/test/fts5rank.test 47c1e8e5d84754ff18e012fdd629776088b5a15de41bdd24957581cf084d8a00 -F ext/fts5/test/fts5rebuild.test 83e72d77636378833233fadc7cb7517a2fa446ea7d1f94dd526ba3e7e104b9f5 +F ext/fts5/test/fts5rebuild.test dc09779fbbe151ab68206a0931c10a611912a7a12c7a85d71c5e48453f2375a5 F ext/fts5/test/fts5restart.test 9af2084b8e065130037b95f05f3f220bb7973903a7701e2c5fb916dff7cf80c5 F ext/fts5/test/fts5rowid.test 8632829fec04996832a4cfb4f0bd89721ba65b7e398c1731741bdb63f070e1a3 F ext/fts5/test/fts5savepoint.test 1447758d7900afe903cef08b4524c5331fb60c1126ae6fba7f4d8704268013c5 @@ -352,51 +352,6 @@ F ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java c7d1452f9ff26175b3c19b F ext/jni/src/tests/000-000-sanity.test c3427a0e0ac84d7cbe4c95fdc1cd4b61f9ddcf43443408f3000139478c4dc745 F ext/jni/src/tests/000-001-ignored.test e17e874c6ab3c437f1293d88093cf06286083b65bf162317f91bbfd92f961b70 F ext/jni/src/tests/900-001-fts.test bf0ce17a8d082773450e91f2388f5bbb2dfa316d0b676c313c637a91198090f0 -F ext/lsm1/Makefile 851a8108af2f00d6086b7be8a76fe54eabe2dd4cfd4171fd39285c5b11bee90d -F ext/lsm1/Makefile.msc f8c878b467232226de288da320e1ac71c131f5ec91e08b21f502303347260013 -F ext/lsm1/lsm-test/README 87ea529d2abe615e856d4714bfe8bb185e6c2771b8612aa6298588b7b43e6f86 -F ext/lsm1/lsm-test/lsmtest.h cf58528ffe0cfe535e91b44584e2ec5fb1caacdabecef0d8dcf83bf83168bf28 -F ext/lsm1/lsm-test/lsmtest1.c 54374fe88cee888c52c31160013c26184288f47a45b23d4d85390aa539733aab -F ext/lsm1/lsm-test/lsmtest2.c 188b09aec776516aeedcfd13b9c6faf85ba16b3671a0897a2c740ee00a5dc4f8 -F ext/lsm1/lsm-test/lsmtest3.c 9ab87528a36dbf4a61d7c8ad954f5ee368c0878c127b84b942b2e2abe522de26 -F ext/lsm1/lsm-test/lsmtest4.c d258d6a245db5d8eaede096e2368d23f859c5e92c80ab9122463f708514fe10c -F ext/lsm1/lsm-test/lsmtest5.c 8d5242a0f870d65eeada191c8945781fed9cb8ece3886573790ebd373b62dac5 -F ext/lsm1/lsm-test/lsmtest6.c 869cb4a172cd07d1a75b3aeaecd61d0a477787b3b8668bad0d3ff0f43b642b7c -F ext/lsm1/lsm-test/lsmtest7.c 7a917455a0f956a8ed3f44f5c9387ec0ea6627714874464cc3fa5c5a9cabb2f2 -F ext/lsm1/lsm-test/lsmtest8.c 773f226163d0f0d62701e3764d0c35fd4365faca74098bd63648bc57d6f14402 -F ext/lsm1/lsm-test/lsmtest9.c 0a168757b757b106191acf43143dbbb5b2d76e57a3c8fd3018cecbaee1080aba -F ext/lsm1/lsm-test/lsmtest_bt.c 79b24bfd37e05fd626c35ec23bc5bb62d8a403afd66c710335384884dc1366d7 -F ext/lsm1/lsm-test/lsmtest_datasource.c 5d770be191d0ca51315926723009b2c25c0b4b8136840494ef710ac324aa916c -F ext/lsm1/lsm-test/lsmtest_func.c 159aa401bc8032bfa3d8cf2977bd687abebab880255895a5eb45770d626fa38d -F ext/lsm1/lsm-test/lsmtest_io.c cf11b27b129c6bd5818fa1d440176502dc27229f0db892b4479118d61993ea20 -F ext/lsm1/lsm-test/lsmtest_main.c a9bc647738c0dcaebf205d6d194b3ce4a6ef3925801cd2d919f0a4ea33a15aeb -F ext/lsm1/lsm-test/lsmtest_mem.c 4e63c764345ab1df59d4f13a77980c6f3643798210b10d6cdbd785b4b888fda5 -F ext/lsm1/lsm-test/lsmtest_tdb.c 754b1ca8e1cfa7b29cbe2e4ab500f7eee0059033741b8d83267afe6f495a536d -F ext/lsm1/lsm-test/lsmtest_tdb.h 8733eee249b12956a9df8322994b43d19bd8c02ad2e8b0bb5164db4d6ccc1735 -F ext/lsm1/lsm-test/lsmtest_tdb2.cc aebe50f2cb7a759214241938046fe5f00da66e4217637f946f436ca209776af9 -F ext/lsm1/lsm-test/lsmtest_tdb3.c 7a7ccae189f5bb25bcd1ec3bbd740529706eded7f6729a5a0a9eeaeb57785320 -F ext/lsm1/lsm-test/lsmtest_tdb4.c cbe230727b9413d244062943371af1421ace472ccb023b75af6540e0fa52b1bb -F ext/lsm1/lsm-test/lsmtest_util.c 241622db5a332a09c8e6e7606b617d288a37b557f7d3bce0bb97809f67cc2806 -F ext/lsm1/lsm-test/lsmtest_win32.c 0e0a224674c4d3170631c41b026b56c7e1672b151f5261e1b4cc19068641da2d -F ext/lsm1/lsm.h 0f6f64ff071471cb87bf98beb8386566f30ea001 -F ext/lsm1/lsmInt.h 8e4ead919951d700c2d065326b22c8bf055b7fec6e75514c6151b5bb5d9d3030 -F ext/lsm1/lsm_ckpt.c 702a08e9fd92695976d759b259e2258330da99b3f8ac845f145e418cb70902ee -F ext/lsm1/lsm_file.c 4bbc4cb1a558089d884e1e5a17b021d9056ae62add32dd6906d070954c7fe954 -F ext/lsm1/lsm_log.c 9450d193db7a50c96805f10f393ac8b08b2009b6330b7df7ae1e4b442ed219a7 -F ext/lsm1/lsm_main.c 87770a9c7e73859fce7620cb79623776ba4b30369086229ad82c3e6eeaf45457 -F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a -F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea -F ext/lsm1/lsm_shared.c c67282a4f2c91e2a3362bdd40a81f9041cd587973ffc4bca8b8fbdab5470dee1 -F ext/lsm1/lsm_sorted.c 1bae85469458793adf753f7c5e695d8d15e33f28bc0ca07ebc9f1b58e853c56c -F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82 -F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb -F ext/lsm1/lsm_unix.c 11e0a5c19d754a4e1d93dfad06de8cc201f10f886b8e61a4c599ed34e334fc24 -F ext/lsm1/lsm_varint.c fe134ad7b2db1ecd99b6a155d2f3625cfd497730e227ae18892452e457b73327 -F ext/lsm1/lsm_vtab.c 0bc7d2702150e9d5513118f23fdb5d7f3642884e6c0dde332da08b016857887a -F ext/lsm1/lsm_win32.c 0a4acbd7e8d136dd3a5753f0a9e7a9802263a9d96cef3278cf120bcaa724db7c -F ext/lsm1/test/lsm1_common.tcl 5ed4bab07c93be2e4f300ebe46007ecf4b3e20bc5fbe1dedaf04a8774a6d8d82 -F ext/lsm1/test/lsm1_simple.test a04d08e8661ae6fc53786c67f0bd102c6692f003e859dde03ed9ac3f12e066e5 -F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f6078e07335398b0 F ext/misc/README.md af13c3bf4405709eb1e2e1e3a39c3be6a15c3189ab8a0642bb107c6eb223b298 F ext/misc/amatch.c 2db45b1499b275d8340af6337a13d6216e4ceb2ddb41f4042b9801be7b5e593d F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb @@ -417,7 +372,7 @@ F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f82 F ext/misc/decimal.c 228d47e9ef4de60daf5851da19e3ac9ac1eda9e94432816914469501db6a1129 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 F ext/misc/explain.c 606100185fb90d6a1eade1ed0414d53503c86820d8956a06e3b0a56291894f2b -F ext/misc/fileio.c 5e1aff478ea12455a76be25dff321e1e9517b189fffb1d547a019625c49c1880 +F ext/misc/fileio.c 88cb2e5744296de6638af02ef6349fd468c2eb5e5f41ba405f88d9b4ad500f8e F ext/misc/fossildelta.c 0aeb099e9627eea693cf21ae47826ecd1e0319b93143bed23090838b2ef0c162 F ext/misc/fuzzer.c 6b231352815304ba60d8e9ec2ee73d4918e74d9b76bda8940ba2b64e8777515e F ext/misc/ieee754.c c9dd9d77c8e8e18e0a5706f8ffcccf4ccb6562073709f7453d4d73f5122f4362 @@ -456,12 +411,12 @@ F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505 F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c 0b23c0a69a2b63dc0ef0af44f9c1fc977300c480a1f7a9814500369d8211f56e F ext/misc/vfstrace.c 0e4b8b17ac0675ea90f6d168d8214687e06ca3efbc0060aad4814994d82b41fb -F ext/misc/vtablog.c a197addbbd1e267a5476274b74953e1b6f050e28516f0a5fe7d6382753165ee6 +F ext/misc/vtablog.c 9f7e02e9e8de585f3bfb48405db36c2eb4b680a23a67d7a4b738dd20f6ad7aec F ext/misc/vtshim.c e5bce24ab8c532f4fdc600148718fe1802cb6ed57417f1c1032d8961f72b0e8f F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 F ext/misc/windirent.h 02211ce51f3034c675f2dbf4d228194d51b3ee05734678bad5106fff6292e60c -F ext/misc/zipfile.c b62147ac4985eaac4e368d529b1f4f43ad6bc9ac13d6805d907fff3afdac64d3 -F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64 +F ext/misc/zipfile.c 360cc8e0b13398a27abae2baa5d136462718994053ef918e86f4e2dd238657c7 +F ext/misc/zorder.c bddff2e1b9661a90c95c2a9a9c7ecd8908afab5763256294dd12d609d4664eee F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 F ext/rbu/rbu10.test 7c22caa32c2ff26983ca8320779a31495a6555737684af7aba3daaf762ef3363 @@ -539,7 +494,7 @@ F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c3350 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 734aa36238bcd2dee91db5dba107d5fcbdb02396612811377a8ad50f1272b1c1 F ext/rtree/geopoly.c f0573d5109fdc658a180db0db6eec86ab2a1cf5ce58ec66cbf3356167ea757eb -F ext/rtree/rtree.c a1f04a2013bc8f982307760615d1cee591355ac4723c74c3761c128d5c6954ab +F ext/rtree/rtree.c 86967c5a501f895b9705366b8cd9c37f15a9ebdff770ceb719c7deeeb2c22b72 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test e0608db762b2aadca0ecb6f97396cf66244490adc3ba88f2a292b27be3e1da3e F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d @@ -557,7 +512,7 @@ F ext/rtree/rtreeD.test fe46aa7f012e137bd58294409b16c0d43976c3bb92c8f710481e577c F ext/rtree/rtreeE.test e65d3fc625da1800b412fc8785817327d43ccfec5f5973912d8c9e471928caa9 F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd450ed331 F ext/rtree/rtreeG.test 1b9ca6e3effb48f4161edaa463ddeaa8fca4b2526d084f9cbf5dbe4e0184939c -F ext/rtree/rtreeH.test 0885151ee8429242625600ae47142cca935332c70a06737f35af53a7bd7aaf90 +F ext/rtree/rtreeH.test c304651ee87dbb60296366d2c802f3043c3d00506c79a5c85bd20f2c9274498b F ext/rtree/rtreeI.test 608e77f7fde9be5a12eae316baef640fffaafcfa90a3d67443e78123e19c4ca4 F ext/rtree/rtreeJ.test 93227ccd4d6c328f5ac46a902b8880041509dd2d68f6ce71560f0d8ab5bb507a F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 @@ -594,6 +549,7 @@ F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401 F ext/session/sessionG.test 3efe388282d641b65485b5462e67851002cd91a282dc95b685d085eb8efdad0a F ext/session/sessionH.test 71bbff6b1abb2c4ac62b84dee53273c37e0b21e5fde3aed80929403e091ef859 +F ext/session/sessionI.test 11e7b6729fc942982a5104a40132f70a2e964d64d60dc5809b8206465af74822 F ext/session/session_common.tcl a31f537a929a695a852d241c9434f2847cadf329856401921139fbb03a5a7697 F ext/session/session_gen.test 942a0002df10da53c45b40b581cc3ed25e7ff42bda1e7ba497273dc2887aa8e6 F ext/session/session_speed_test.c dcf0ef58d76b70c8fbd9eab3be77cf9deb8bc1638fed8be518b62d6cbdef88b3 @@ -617,11 +573,11 @@ F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544a806421b85dc2dec F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c 6b0877fe1ab832aa4b85eaca72606dfd1630a1363a1be7af10ee1042a5ec719e -F ext/session/sqlite3session.h 9bb1a6687b467764b35178dc29bbd2c57ab8cd3acdc8a62f088c34ad17e4fe2b -F ext/session/test_session.c 2ddff73ea368d827028c32851b291416e1008845832feb27b751d15e57e13cc3 +F ext/session/sqlite3session.c 19e14bcca2fbc63a8022ffd708ea6e6986c4003a1e9bbca9b2989fd230362e15 +F ext/session/sqlite3session.h b81e8536ce4b83babafd700f4ff67017804b6c1d71df963b30d3972958e7f4a7 +F ext/session/test_session.c 8766b5973a6323934cb51248f621c3dc87ad2a98f023c3cc280d79e7d78d36fb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile e315b903c33b28ca074367041849de9dd2681c96d1437c11a5e9596abc265012 +F ext/wasm/GNUmakefile 35e730a01b32481f5483ea5bd72c3d4609e25f34cb5aab9f85eb3eba6f0c4935 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md b89605f65661cf35bf034ff6d43e448cc169b8017fc105d498e33b81218b482c F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -630,7 +586,7 @@ F ext/wasm/SQLTester/SQLTester.run.mjs 57f2adb33f43f2784abbf8026c1bfd049d8013af1 F ext/wasm/SQLTester/index.html 64f3435084c7d6139b08d1f2a713828a73f68de2ae6a3112cbb5980d991ba06f F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core 2bcbbfe3b95c043ed6037e2708a2ee078d212dd1612c364f93588d8dc97300fe -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras fe40d6d758646e38f8b15f709044951e10884214f5453d35502100179c388c13 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras cb4fa8842c875b6ee99381523792975c5ebb7371bd27fbd1bd863a43c7f3505a F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md c64ec8e84449c069e0217706d9d7d31b3bd53627228b2ba0c3cddbdc2350ca66 @@ -640,18 +596,18 @@ F ext/wasm/api/post-js-footer.js 365405929f41ca0e6d389ed8a8da3f3c93e11d3ef43a90a F ext/wasm/api/post-js-header.js 53740d824e5d9027eb1e6fd59e216abbd2136740ce260ea5f0699ff2acb0a701 F ext/wasm/api/pre-js.c-pp.js a614a2c82b12c4d96d8e3ba77330329efc53c4d56a8a7e60ade900f341866cfb F ext/wasm/api/sqlite3-api-cleanup.js 3ac1786e461ada63033143be8c3b00b26b939540661f3e839515bb92f2e35359 -F ext/wasm/api/sqlite3-api-glue.c-pp.js bd8ae059bb3ea489666b2f167025833e55dc5941be99c129a56fc25e541d37ce -F ext/wasm/api/sqlite3-api-oo1.c-pp.js f3a8e2004c6625d17946c11f2fb32008be78bc5207bf746fc77d59848813225f -F ext/wasm/api/sqlite3-api-prologue.js 8708570165f5b4bce9a78ccd91bc9ddf8735970ac1c4d659e36c9a7d9a644bb4 -F ext/wasm/api/sqlite3-api-worker1.c-pp.js f646a65257973b8c4481f8a6a216370b85644f23e64b126e7ae113570587c0ab +F ext/wasm/api/sqlite3-api-glue.c-pp.js 0b76510f3650053bac67ca8947cb6ab9d050ad2218118a2e7796dd37be832ffa +F ext/wasm/api/sqlite3-api-oo1.c-pp.js 852f2cd6acddbae487fc4f1c3ec952e6c1e2033aa4e6c7091d330d983c87c032 +F ext/wasm/api/sqlite3-api-prologue.js 4f1c2a9dc9caf631907766e9872c27d11b255ccae779e8af01c7f8b932817214 +F ext/wasm/api/sqlite3-api-worker1.c-pp.js 760191cd13416e6f5adfd9fcc8a97fed5645c9e0a5fbac213a2d4ce2d79a4334 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 0f68a64e508598910e7c01214ae27d603dfc8baec6a184506fafac603a901931 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4ab0704ee198de7d1059eccedc7703c931510b588d10af0ee36ea5b3ebbac284 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616 -F ext/wasm/api/sqlite3-wasm.c 7ea3d4a286a2241f6fcc65c9ff10fc04ee5590f80f40763a57001dd5e93aa4c4 -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc65debfe43b81fc39fb25c40ad0cc1946bd82580fbf644351107b544d6177ee +F ext/wasm/api/sqlite3-wasm.c 292db8be42a6ba29a7df2cc2723553a47627853d7c1fdcf8fa37743cf5d85497 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 4ad256b4ff7f839ad18931ed35d46cced544207bd2209665ec552e193f7f4544 F ext/wasm/api/sqlite3-worker1.c-pp.js 5e8706c2c4af2a57fbcdc02f4e7ef79869971bc21bb8ede777687786ce1c92d5 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd @@ -661,8 +617,8 @@ F ext/wasm/c-pp.c cca55c5b55ebd8d29916adbedb0e40baa12caa9a2e8429f812683c308f9b0e F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f -F ext/wasm/common/whwasmutil.js 5e7fe4a2547831e003356c201ca28d9a0d6a19ab1fe2c8f63f82464fecd2cef7 -F ext/wasm/config.make.in 4bc43443f768a61efd43cf995a5e618f58ac9afc0936706014193537d82c41cb +F ext/wasm/common/whwasmutil.js ea50e847cf08c2a96694f87d00b49fb97ee5fe62c15439cc269c8e1d9ab74c0a +F ext/wasm/config.make.in c424ae1cc3c89274520ad312509d36c4daa34a3fce5d0c688e5f8f4365e1049a F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf @@ -672,25 +628,25 @@ F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a F ext/wasm/demo-worker1-promiser.c-pp.js af168699d3cab1c27ad2364ebe06cd49db300bdbf404e23b00d5742ed52816ba F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js 08720227e98fa5b44761cf6e219269cee3e9dd0421d8d91459535da776950314 -F ext/wasm/dist.make 92ef4ffe33022a50f92d602acabad10bd8dd91759f3eb7df27fc6d7d37072b96 +F ext/wasm/dist.make c29018b4db479a4c170569393e5399f0625446123a7eb6ffb0677495292bb954 F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle.make c6d7a3d6cc03bb5f21acb295c1233820d0dbf5c6a89b28dc2e093edcc001c45a -F ext/wasm/fiddle/fiddle-worker.js 850e66fce39b89d59e161d1abac43a181a4caa89ddeea162765d660277cd84ce -F ext/wasm/fiddle/fiddle.js 2a2f27b4be2674f501fff61c4a09e44dcf2295731a26b5c28e439f3a573bd269 -F ext/wasm/fiddle/index.html 7fcfb221165183bef0e05d5af9ceb79b527e799b1708ab05de0ec0eaebd5b7bf +F ext/wasm/fiddle.make ea505d11aa2a89551e1693ed4c71ee6a163364ca14f806dda295d0beb26ec0ea +F ext/wasm/fiddle/fiddle-worker.js 50d3edf54c0c0e3657e876724ec2c10069f55f3e40af20864d72f6f6e9ad00f8 +F ext/wasm/fiddle/fiddle.js f0b96f978c7c77fea8d092aa79c77849ce111d7b1ba60ffba07675009682184e +F ext/wasm/fiddle/index.html 17c7d6b21f40fbf462162c4311b63d760b065e419d9f5a96534963b0e52af940 F ext/wasm/index-dist.html 56132399702b15d70c474c3f1952541e25cb0922942868f70daf188f024b3730 F ext/wasm/index.html bcaa00eca521b372a6a62c7e7b17a870b0fcdf3e418a5921df1fd61e5344080d F ext/wasm/jaccwabyt/jaccwabyt.js 6e4f26d0edb5c2e7d381b7eff1924832a040a12274afab2d1e1789027e9f6c5c F ext/wasm/jaccwabyt/jaccwabyt.md 1128e3563e7eff90b5a373395251fc76cb32386fad1fea6075b0f34a8f1b9bdf -F ext/wasm/mkwasmbuilds.c 5b096a3c9fdf6e67eb20329dc685f995e820248a67fa970633c2c8f49bf472ad +F ext/wasm/mkwasmbuilds.c cc66cfaf8673ece3c30ca7fe28f6111481090648098a143ea619a8820b8fbe82 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs c77c7231338ed5c0e1ce16aa29106df8e5b5cf11a48319c49433490a8d3ded30 -F ext/wasm/speedtest1-worker.html 864b65ed78ce24847a348c180e7f267621a02ca027068a1863ec1c90187c1852 +F ext/wasm/speedtest1-worker.html d24d1e06caf3dcd83430c8c3d87761ff7555fd06eaeaf2fc02ce49cf45f0d032 F ext/wasm/speedtest1-worker.js 95e549e13a4d35863a9a7fc66122b5f546c0130d3be7b06dfcc556eb66d24bde -F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da +F ext/wasm/speedtest1.html e2a0e0bd12243ca34b11235bf9f3c229f4574ea1125f2ecf2bf0589853d6f9c8 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 @@ -698,7 +654,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 419717b16e12703487a7ccf3ea4e63d693bdfbf7657e55a7e6c559bbccf027d3 +F ext/wasm/tester1.c-pp.js 0abba4bd54f6b22adaadf836c04d3163399f7a8a490fd60f20daac5f9c42b47d F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -707,9 +663,9 @@ F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36e0f6991460fff0cb7c15079454679a4e2 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 -F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 +F ext/wasm/wasmfs.make 411dd94b40406572caddf88392a1ccc4deed0f88d260516e59ca6e0c887ee861 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 3ced12d068b22957f4e82a0b94a2c1d83b37dcf39c087263691267f7bfb39a84 +F main.mk a5d698cf8d38e4eb42a89f08a9521906b24f4acac61bbafa56d72cd9b39617b6 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -717,8 +673,8 @@ F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 F mptest/mptest.c aa41ace6dbc5050d76b02548d3521e6bbccae4f0 F mptest/multiwrite01.test dab5c5f8f9534971efce679152c5146da265222d F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b -F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc -F sqlite3.pc.in 0977c03a4da7c4204bd60e784a0efb8d51a190448aba78a4e973fe7192bdaf03 +F sqlite3.1 1b9c24374a85dfc7eb8fa7c4266ee0db4f9609cceecfc5481cd8307e5af04366 +F sqlite3.pc.in e6dee284fba59ef500092fdc1843df3be8433323a3733c91da96690a50a5b398 F src/alter.c fc7bbbeb9e89c7124bf5772ce474b333b7bdc18d6e080763211a40fde69fb1da F src/analyze.c 03bcfc083fc0cccaa9ded93604e1d4244ea245c17285d463ef6a60425fcb247d F src/attach.c 9af61b63b10ee702b1594ecd24fb8cea0839cfdb6addee52fba26fa879f5db9d @@ -726,17 +682,17 @@ F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c e242d4496774dfc88fa278177dd23b607dce369ccafb3f61b41638eea2c9b399 F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea -F src/btree.c 53a9c7b243e94c992853d90f1dac0959028433b4b0d27e04409cee04e001b508 -F src/btree.h 18e5e7b2124c23426a283523e5f31a4bff029131b795bb82391f9d2f3136fc50 +F src/btree.c cb5b8ceb9baa02a63a2f83dec09c4153e1cfbdf9c2adef5c62c26d2160eeb067 +F src/btree.h e823c46d87f63d904d735a24b76146d19f51f04445ea561f71cc3382fd1307f0 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 -F src/build.c 67c1db4c5e89a8519fe9b6dafc287f6bc3627696b5b8536dc5e06db570d8c05f +F src/build.c cc4f287348790bbb7219f7e8dee13b1c345c3377fcdd98eca866e7457ecd07e7 F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9 F src/dbpage.c b3e218f8ed74fcbb7fa805df8ca669a3718d397617b3d8a8aac3307dc315c4d6 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c f16fa5cbd849991462edf1d31bb7def5b970bb9611afcb4ea21c77e88e52a220 +F src/expr.c 12aeb13773920b48831d7b53018d5cc79e47b3bd8ae7c0fdfd28e6aab977821a F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c de47a8295503aa130baae5e6d9868ecf4f7c4dbffa65d83ad1f70bdbac0ee2d6 @@ -770,28 +726,28 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 04e054ab86d86a7be99ebe5265922687791a40df5afc781d059beb47f4a40acd -F src/os_win.c b8d3cfdf2f40e2f9715b7d8df64f3c0c7ee18743a2dd0c4fc70c1d57fa1aadc7 +F src/os_unix.c 690107e26cc4e9809eeb9826c0efdbff4a42b9cc59d0f0b855ca3e6021e1ae73 +F src/os_win.c 7ac69df49d2ff0432b9c96fd2d9a17a100cced6860479e584cd3337e18d09334 F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19 F src/pager.c 23c0f17deb892da6b32fef1f465507df7ab5cd01d774288cb43695658a649259 F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8 -F src/parse.y e426d7323311554c75b0aebc426d0fe3c88d9777ffefed236f343ad9e661dc4c +F src/parse.y 619c3e92a54686c5e47923688c4b9bf7ec534a4690db5677acc28b299c403250 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd F src/pragma.c 30b535d0a66348df844ee36f890617b4cf45e9a22dcbc47ec3ca92909c50aaf1 F src/prepare.c 1832be043fce7d489959aae6f994c452d023914714c4d5457beaed51c0f3d126 -F src/printf.c 71b6d3a0093bf23f473e25480ca0024e8962681506c75f4ffd3d343a3f0ab113 +F src/printf.c 5f0c957af9699e849d786e8fbaa3baab648ca5612230dc17916434c14bc8698f F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c d40fe18d7c2fd0339f5846ffcf7d6809866e380acdf14c76fb2af87e9fe13f64 +F src/resolve.c f8d1d011aba0964ff1bdccd049d4d2c2fec217efd90d202a4bb775e926b2c25d F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 882d739e0d5e6c7a8b46a3cca3ada37fe1a56301f1360d6b141312c666bbe482 -F src/shell.c.in 4ae9ff8e8b23d9a309e50d5b5ef2768b9bb18f748ece2acc2641a3c23b71e2b9 -F src/sqlite.h.in 22882ddd3a70751aa8864c81993ee4562ed54c2c508b6270f75e223ffee38e1b +F src/select.c a6be657216e1fb72f85dad7df0dba0eb79fe76527c08caa65da8fe44f0e4db44 +F src/shell.c.in 7918c9355667b3b348e5850f0dad9095476ef942ee3b96ee9b8bc2710adda1da +F src/sqlite.h.in b526a1eaa60096c9c043d7b128daf2764571e77413873888ee5582ca0141804c F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 F src/sqlite3ext.h 0bfd049bb2088cc44c2ad54f2079d1c6e43091a4e1ce8868779b75f6c1484f1e -F src/sqliteInt.h 60172f8b15efb8e9ae37bcb5ce596f1c55fee01b555b80f1c9056e55a6ee7d28 -F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b +F src/sqliteInt.h a54f83985985655d1276e9e356dd6ae19b8d0b62c2abc75cc9e8f402ea141207 +F src/sqliteLimit.h fe70bd8983e5d317a264f2ea97473b359faf3ebb0827877a76813f5cf0cdc364 F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 3c604c49e6cf4211960a9ddb9505280fd22cde32175f40884c641c0f5a286036 @@ -844,20 +800,20 @@ F src/test_vfs.c b4135c1308516adf0dfd494e6d6c33114e03732be899eace0502919b674586b F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 3e37ac2b6cbb9b0abe33827b0153c27595269afd7152b48019808974481aca2c +F src/tokenize.c 8400646d2830afc2f2dc465a75e3a92e4bedeea623f19dbd79c0c12d0dd6dda2 F src/treeview.c d85ce76e6d1498d781957c07cb234da6d77ce0ed2d196480d516f54dabc62279 -F src/trigger.c 3ffb8ed6b64dbcc0ccae6e82435d01be3bf547e13b814e2d46f7df9bef84748e +F src/trigger.c d5cf2541ff048f30b6a0507eb3d1ec4e695c53584e3b2298a5bf248714fe185e F src/update.c 3e5e7ff66fa19ebe4d1b113d480639a24cc1175adbefabbd1a948a07f28e37cf F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 7267c3fb9e2467020507601af3354c2446c61f444387e094c779dccd5ca62165 F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 F src/vacuum.c 1bacdd0a81d2b5dc1c508fbf0d938c89fa78dd8d5b46ec92686d44030d4f4789 -F src/vdbe.c 714fab7aa7c516edbcf5e4f653ae8f548a3e24c0ed19086d7383bb5851983992 -F src/vdbe.h 93761ed7c6b8bc19524912fd9b9b587d41bf4f1d0ade650a00dadc10518d8958 -F src/vdbeInt.h 0bc581a9763be385e3af715e8c0a503ba8422c2b7074922faf4bb0d6ae31b15e -F src/vdbeapi.c 613a6f29efacd6ed83e886b6e52db0fe52ba80a596b0a137608db1948bad90a9 -F src/vdbeaux.c fd2c6b19a8892c31a2adc719f156f313560f9cc490cdbd04ff08fdae5d7aedb7 -F src/vdbeblob.c b1b4032cac46b41e44b957c4d00aee9851f862dfd85ecb68116ba49884b03dfd +F src/vdbe.c a5873cd566a0e2a0344a86dd946add9d34fae3feeae8b126277ef7af8dc11f91 +F src/vdbe.h ea1f1b52f0efe422f80d88da3c57e4eadc72856e29a22f1ff08e502ec6ba5f08 +F src/vdbeInt.h 52896dd4d5b62190c53db14b09fc2484434eb594c963df0fa66eb8a94527b02e +F src/vdbeapi.c f9a4881a9674fec3fa13da35044a1484d3c4b95f9ec891cc8ffb02ef2b7a41df +F src/vdbeaux.c b701e5920fe74b907eb8211d1f63fef96adc65dfd6e1ad6ed0843c71d8c65205 +F src/vdbeblob.c b3f0640db9642fbdc88bd6ebcc83d6009514cafc98f062f675f2c8d505d82692 F src/vdbemem.c e67d9c6484d868c879d20c70d00bf4a9058082f1d4058607ca15d50eb3aebc21 F src/vdbesort.c cb6f472e83ca12c46aa7de0ac0a9d11458b357986f2617a1c90dfb19a542ecbe F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 @@ -867,10 +823,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 20be6f0a25a80b7897cf2a5369bfd37ef198e6f0b6cdef16d83eee856056b159 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c a99fa3061a0155d2cb0e2c91df76dbf834750272a8d79ec5e2dce3ed4e6abad6 -F src/whereInt.h 02b646ea41a8342815b3628f8064c32618ea2e0f20b83216ea08cad11f0ac5aa -F src/wherecode.c 9710e62379c000189476404f923d4d1b192d0def222fdd287b820cc085a0d555 -F src/whereexpr.c 566ca4382e07a4ba1fd86c97ae0781cdf84004c7d9c59466bf5db75733548807 +F src/where.c f2f075bd17065922235632feb368efe92a7f03d42797eb575267574fbf6d4218 +F src/whereInt.h 8d94cb116c9e06205c3d5ac87af065fc044f8cf08bfdccd94b6ea1c1308e65da +F src/wherecode.c 71c5c6804b7f882dec8ec858758accae02fcfca13df3cc720f1f258e663ec7c5 +F src/whereexpr.c 78c28a8da187816d5d82049f2e343fb39f4a8e30b5bf1bda9b96cecde40ca8bd F src/window.c d01227141f622f24fbe36ca105fbe6ef023f9fd98f1ccd65da95f88886565db5 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test 4d7a34d328e58ca2a2d78fd76c27614a41ca7ddf4312ded9c68c04f430b3b47d @@ -898,8 +854,8 @@ F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584 F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd436f076e81 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 -F test/altertab2.test 4bad0fa9b1ad6e62d07bc2ddb0807fb98ba80ee06d6593db2e514ec1821cae3a -F test/altertab3.test b331ae34e69594e19605e3297805202d6156fcc8f75379dfd972a2e51cae8721 +F test/altertab2.test 0889ba0700cc1cdb7bc7d25975aa61fece34f621de963d0886e2395716b38576 +F test/altertab3.test 471b8898d10bbc6488db9c23dc76811f405de6707d2d342b1b8b6fd1f13cd3c8 F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b @@ -962,9 +918,9 @@ F test/bestindex8.test b63a4f171a2c83d481bb14c431a8b72e85d27b2ffdaa0435a95d58ca9 F test/bestindex9.test 1a4b93db117fd8abe74ae9be982f86aa72f01e60cd4ac541e6ede39673a451a0 F test/bestindexA.test e1b5def6b190797cacf008e6815ffb78fb30261999030d60a728d572eef44c7f F test/bestindexB.test 328b97b69cd1a20928d5997f9ecb04d2e00f1d18e19ab27f9e9adb44d7bc51ce -F test/bestindexC.test 2df6ada16d8f00d9bb6a9664d9c323560aeed0e0ebc7a32b99d85d70037fd250 +F test/bestindexC.test 95b4a527b1a5d07951d731604a6d4cf7e5a806b39cea0e7819d4c9667e11c3fc F test/bestindexD.test 6a8f6f84990bcf17dfa59652a1f935beddb7afd96f8302830fbc86b0a13df3c3 -F test/between.test b9a65fb065391980119e8a781a7409d3fcf059d89968279c750e190a9a1d5263 +F test/between.test e7587149796101cbe8d5f8abae8d2a7b87f04d8226610aa1091615005dcf4d54 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc F test/bigmmap.test 6021e205487347c6d7e5a541aa472a4b8efc4e9f4a3799a823b61a8e6616105d @@ -1131,7 +1087,7 @@ F test/enc.test b5503a87b31cea8a5084c6e447383f9ca08933bd2f29d97b6b6201081b2343eb F test/enc2.test 872afe58db772e7dfa1ad8e0759f8cc820e9efc8172d460fae83023101c2e435 F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 82f221e8cd588434d7f3bba9a0f4c78cbe7a541615a41632e12f50608bfb4a99 +F test/eqp.test 746db9fe11629a0d00328e1721cc2a2e4726d574b677ab14de35fd914f54cc82 F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 F test/errofst1.test 6da78363739ba8991f498396ab331b5d64e7ab5c4172c12b5884683ef523ac53 @@ -1140,6 +1096,9 @@ F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650 F test/exclusive2.test cd70b1d9c6fffd336f9795b711dcc5d9ceba133ad3f7001da3fda63615bdc91e F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac +F test/existsexpr.test 9c4b77c4729281cc2ae63b9b460d0598ce28cc7876135e3e2c21629bbc8d077a +F test/existsexpr2.test dc23e76389eff3d29f6488ff733012a3560cd67ec8cfaecbecd52cced5d5af11 +F test/existsfault.test ff41c11f3052c1bbd4f8dd557802310026253d67d7c4e3a180c16d2f0862973e F test/expr.test 4ada8eb822c45ef27a36851a258004d43c1e95e7c82585a1217e732084e4482c F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 F test/exprfault.test da33606d799718e2f8e34efd0e5858884a1ad87f608774c552a7f5517cc27181 @@ -1281,7 +1240,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 19f8af47a5c4ee2c3943fdee270f1f14e3d83fe968a9737a7557fb4e3c06efc1 +F test/fuzzcheck.c da9767e7cbb8da0a06cb4e9df03c9d3d388160b4c59200013cebef338880cd5d F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1300,7 +1259,7 @@ F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/having.test a89236dd8d55aa50c4805f82ac9daf64d477a44d712d8209c118978d0ca21ec9 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 3481a68009fe143e3363fca922f6fc7a1e1f3776c51e42777f1a01b26ad2a9c8 +F test/hook.test 2d89bf9480646feb8093be3a58ea502d6521906779ed960de31dd9c4502c0541 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e @@ -1315,7 +1274,7 @@ F test/in7.test d9efdee00b074a60c6343993b2eda78bc369ab080dad864513c73f8aca89d566 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f F test/incrblob3.test 67621a04b3084113bf38ce03797d70eca012d9d8f948193b8f655df577b0da6f -F test/incrblob4.test 21a52a6843a56cdcce968c6a86b72a7066d0e6ba +F test/incrblob4.test a8d6b5ff04055fcec747a50b78485ebf4fcd80074e0b3cedbe952bde346da54a F test/incrblob_err.test 89372a28f1d98254f03fed705f9efcd34ef61a674df16d2dbb4726944a2de5e9 F test/incrblobfault.test de274b1e329169c2c3438f9528994807ea8201ebf38ae9f157d34bf3ec0cc549 F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a @@ -1387,7 +1346,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh 7d5898808ce7542762318306ae6075a30f5e7ee115c4a409f487e123afe91d88 x F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7 -F test/json101.test 8237a484c256965eab1678fd950a32ac56325bb7d0dadbd095a46b0ddd95d62b +F test/json101.test cf53254f0f0c1399a01b21fc58fee0e63a12a556be91b9ee9faccdb8b82c083c F test/json102.test 9b2e5ada10845ff84853b3feaae2ce51ce7145ae458f74c6a6cecc6ef6ee3ae1 F test/json103.test 355746a6b66aa438f214b4fae454b13068fad2444b5f693e0d538ad1c059b264 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -1494,7 +1453,7 @@ F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 796c7b7157f55c93b4e672b724e9c923a6fc6aa72ac419379a623e2350472e22 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18 -F test/notnull2.test 2ac7b4e04917148c7a1a9ed36df20150175ce942f07f5714375b29acbaca7106 +F test/notnull2.test 5b7dd6e82c409b2d011ad6acf19ae4bf0816a9c69ccf600b529d7405d7c49874 F test/notnullfault.test fc4bb7845582a2b3db376001ef49118393b1b11abe0d24adb03db057ee2b73d5 F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6f F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9 @@ -1529,7 +1488,7 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 -F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035ce4b3 +F test/parser1.test 131f4733472252d53d8ed681115257866f55740ab697fa05900d766049348f27 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/pendingrace.test e99efc5ab3584da3dfc8cd6a0ec4e5a42214820574f5ea24ee93f1d84655f463 @@ -1576,7 +1535,7 @@ F test/round1.test 29c3c9039936ed024d672f003c4d35ee11c14c0acb75c5f7d6188ff16190c F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc8e -F test/rowvalue.test 9c873b2f6e7ce72b24ef133f93515c07a6a7dac4846a344ebc2af7b8bfdf5147 +F test/rowvalue.test 93474d8e1c496e970bdcc3a7f54ac835adda667d2fd971957b4bce0c0b11707b F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 103e9a224ca0548dd0d67e439f39c5dd16de4200221a333927372408c025324c F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed @@ -1686,7 +1645,7 @@ F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c F test/speedtest.md ee958457ae1b729d9715ae33c0320600000bf1d9ddea1a88dcf79f56729d6fad F test/speedtest.tcl 405411356fbc54af15987b7ffeec330a49138f71584220fb8fe1948b2f7ac907 x -F test/speedtest1.c 64b8804b053a796eab22f8b23fb181000f05d7b3e2aa44f022117ea543bc5a2a +F test/speedtest1.c a9b002a7bfed99ba3166c2a9b8ae45a95b4c2d37f891e1637c022f9e1d15e3f9 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1701,7 +1660,7 @@ F test/statfault.test 064f43379e4992b5221b7d9ac887c313b3191f85cce605d78e416fc404 F test/stmt.test 54ed2cc0764bf3e48a058331813c3dbd19fc1d0827c3d8369914a5d8f564ec75 F test/stmtrand.test 340e2ea4841c5cdc02d36e33739769c5d907ab529b12bb535407def0e413ca17 F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5 -F test/strict1.test 62db60132bb286429a36cd69a218184d78923990b066f131862ecb648105cee0 +F test/strict1.test a7f9091603fe71cdc62baab0766684cba12a97ec69bfbb70be965532669cd77a F test/strict2.test b22c7a98b5000aef937f1990776497f0e979b1a23bc4f63e2d53b00e59b20070 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49 F test/subquery.test 23087f9b1c15ab9cc5231d04946bdebc51db527c95eb9d7434a2222127e17a84 @@ -1733,8 +1692,9 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl 614c4a28f7f730acd7bec53e17d76602fb480e0d538b6ec548169e03a093f92d x +F test/testrunner.tcl b42f7736968cafc9e69bb5d0b87fc81b375fb4c3f44e19b472cccd91d41a416a x F test/testrunner_data.tcl 02dd645b647d907c959fbf232b7ff7d869c2ae430d5117443fc1e16a0d32243a +F test/testrunner_estwork.tcl 7927a84327259a32854926f68a75292e33a61e7e052fdbfcb01f18696c99c724 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1989,7 +1949,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test 2a143fe75a11275781d1fd1988d86b66a3f69cb98f4add62e3da8fd0f637b45f F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test a9417d92111629e2c33e0f3b06cc84bcd611753d0fc017b3ce6ae96572bc2f2f +F test/vtabH.test 3fe4c1cbc93a85971990f379116255e2d4f14b583be4d0b9cca67e90226041f1 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtabJ.test a6aef49d558af90fae10565b29501f82a95781cb4f797f2d13e2d19f9b6bc77b F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3afbf @@ -2152,7 +2112,7 @@ F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkamalzip.tcl 8aa5ebe7973c8b8774062d34e15fea9815c4cc2ceea3a9b184695f005910876a -F tool/mkautoconfamal.sh 564378ae48cc8f4c8c68ef6d00228a0b2a70e2e3e4c67f26be1dd05d9730fefd +F tool/mkautoconfamal.sh cf72166369d9d1df9df1ae9fcad03a7939808d7273784b56fd78b913587637a1 F tool/mkccode.tcl c42a8f8cf78f92e83795d5447460dbce7aaf78a3bbf9082f1507dc71a3665f3c x F tool/mkctimec.tcl 11c9eda4a8d18c74b79280b30506d832849fd1855e6d9e95e1fd506f1d211c37 x F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559 @@ -2168,7 +2128,7 @@ F tool/mksqlite3c.tcl f11b63445c4840509248bd4aa151a81aea25d5415fef71943c8d436eba F tool/mksqlite3h.tcl 989948c6a26e188e673d7c2f2f093ea3acd816ad6ac65bab596280075c8f3a45 F tool/mksqlite3internalh.tcl 46ef6ed6ccd3c36e23051109dd25085d8edef3887635cea25afa81c4adf4d4db F tool/mksrczip.tcl 81efd9974dbb36005383f2cd655520057a2ae5aa85ac2441a80c7c28f803ac52 -F tool/mktoolzip.tcl 34b4e92be544f820e2cc26f143f7d5aec511e826ec394cc82969a5dcf7c7a27c +F tool/mktoolzip.tcl c9f388b2b0751982aef29a0c1d80f7959dbe38065ebb4421c39844e92622b086 F tool/mkvsix.tcl 67b40996a50f985a573278eea32fc5a5eb6110bdf14d33f1d8086e48c69e540a F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c6115285845 F tool/omittest-msvc.tcl d6b8f501ac1d7798c4126065030f89812379012cad98a1735d6d7221492abc08 @@ -2208,8 +2168,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 986b601db11a041d280eb61004723604bf4b6fd573b9832c97e3a9da9ea16e9a -R c89b9ea3a77e2b240253ae60cc2ec713 +P 6bb717acf706e6ffd4671660ca78237e6a42863f344518e6d21065bf735f971e +R 445eb33101f9b4070d3e8b64677babdc U drh -Z 41ec114b4104be2c73f80a11e6cae501 +Z e2734277847ebf981ba375e5cddb7301 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 97bda77ac..2ae98a2c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e9279bff0482806f86657ae05ca3e916708d138bc3c3ceb3fbf454818649d44 +ae9d7c9c922bb241363aa690b42b9664c4ad6e76ed5ce474daf1ab44461bc6a3 @@ -137,17 +137,37 @@ continue prompt = " ...> " .sp .fi -o If the file +o If the environment variable XDG_CONFIG_HOME is set then .B ${XDG_CONFIG_HOME}/sqlite3/sqliterc -or +is checked, else +.B ~/.config/sqlite3/sqliterc +is checked. If the selected file does not exist then the fallback of .B ~/.sqliterc -exists, the first of those to be found is processed during startup. -It should generally only contain meta-commands. +is used. It should generally only contain meta-commands. o If the -init option is present, the specified file is processed. o All other command line options are processed. +.SH HISTORY FILE +.B sqlite3 +may be configured to use a history file to save SQL statements and +meta-commands entered interactively. These statements and commands can be +retrieved, edited and, reused at the main and continue prompts. If the +environment variable +.B SQLITE_HISTORY +is set, it will be used as the name of the history file, whether it +already exists or not. If it is not set but the XDG_STATE_HOME +environment variable is then +.B ${XDG_STATE_HOME}/sqlite_history +is used. If XDG_STATE_HOME is not set then +.B ~/.local/state/sqlite_history +is used. If the selected file does not exist then +.B ~/.sqlite_history +will be used as the history file. If any history file is found, it +will be written if the shell exits interactive mode normally, +regardless of whether it existed previously, though saving will +silently fail if the history file's directory does not exist. .SH SEE ALSO https://sqlite.org/cli.html .br diff --git a/sqlite3.pc.in b/sqlite3.pc.in index a9f941b1e..723dd5156 100644 --- a/sqlite3.pc.in +++ b/sqlite3.pc.in @@ -9,5 +9,5 @@ Name: SQLite Description: SQL database engine Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lsqlite3 -Libs.private: @LDFLAGS_MATH@ @LDFLAGS_ZLIB@ @LDFLAGS_ICU@ +Libs.private: @LDFLAGS_MATH@ @LDFLAGS_ZLIB@ @LDFLAGS_DLOPEN@ @LDFLAGS_PTHREAD@ @LDFLAGS_ICU@ Cflags: -I${includedir} diff --git a/src/btree.c b/src/btree.c index f53060e7f..a931b0d12 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1929,10 +1929,10 @@ static int freeSpace(MemPage *pPage, int iStart, int iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iEnd <= (int)pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); + assert( CORRUPT_DB || iStart<=(int)pPage->pBt->usableSize-4 ); /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. @@ -3074,6 +3074,10 @@ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ sqlite3BtreeEnter(p); pBt->nReserveWanted = (u8)nReserve; x = pBt->pageSize - pBt->usableSize; + if( x==nReserve && (pageSize==0 || (u32)pageSize==pBt->pageSize) ){ + sqlite3BtreeLeave(p); + return SQLITE_OK; + } if( nReserve<x ) nReserve = x; if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); @@ -5663,6 +5667,30 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } +/* Set *pRes to 1 (true) if the BTree pointed to by cursor pCur contains zero +** rows of content. Set *pRes to 0 (false) if the table contains content. +** Return SQLITE_OK on success or some error code (ex: SQLITE_NOMEM) if +** something goes wrong. +*/ +int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + if( pCur->eState==CURSOR_VALID ){ + *pRes = 0; + return SQLITE_OK; + } + rc = moveToRoot(pCur); + if( rc==SQLITE_EMPTY ){ + *pRes = 1; + rc = SQLITE_OK; + }else{ + *pRes = 0; + } + return rc; +} + #ifdef SQLITE_DEBUG /* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that ** this flags are true for a consistent database. @@ -5882,8 +5910,8 @@ moveto_table_finish: } /* -** Compare the "idx"-th cell on the page the cursor pCur is currently -** pointing to to pIdxKey using xRecordCompare. Return negative or +** Compare the "idx"-th cell on the page pPage against the key +** pointing to by pIdxKey using xRecordCompare. Return negative or ** zero if the cell is less than or equal pIdxKey. Return positive ** if unknown. ** @@ -5898,12 +5926,11 @@ moveto_table_finish: ** a positive value as that will cause the optimization to be skipped. */ static int indexCellCompare( - BtCursor *pCur, + MemPage *pPage, int idx, UnpackedRecord *pIdxKey, RecordCompare xRecordCompare ){ - MemPage *pPage = pCur->pPage; int c; int nCell; /* Size of the pCell cell in bytes */ u8 *pCell = findCellPastPtr(pPage, idx); @@ -6012,14 +6039,14 @@ int sqlite3BtreeIndexMoveto( ){ int c; if( pCur->ix==pCur->pPage->nCell-1 - && (c = indexCellCompare(pCur, pCur->ix, pIdxKey, xRecordCompare))<=0 + && (c = indexCellCompare(pCur->pPage,pCur->ix,pIdxKey,xRecordCompare))<=0 && pIdxKey->errCode==SQLITE_OK ){ *pRes = c; return SQLITE_OK; /* Cursor already pointing at the correct spot */ } if( pCur->iPage>0 - && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0 + && indexCellCompare(pCur->pPage, 0, pIdxKey, xRecordCompare)<=0 && pIdxKey->errCode==SQLITE_OK ){ pCur->curFlags &= ~(BTCF_ValidOvfl|BTCF_AtLast); @@ -6236,7 +6263,7 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ n = pCur->pPage->nCell; for(i=0; i<pCur->iPage; i++){ - n *= pCur->apPage[i]->nCell; + n *= pCur->apPage[i]->nCell+1; } return n; } @@ -8693,7 +8720,12 @@ static int balance_nonroot( ** of the right-most new sibling page is set to the value that was ** originally in the same field of the right-most old sibling page. */ if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ - MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + MemPage *pOld; + if( nNew>nOld ){ + pOld = apNew[nOld-1]; + }else{ + pOld = apOld[nOld-1]; + } memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); } diff --git a/src/btree.h b/src/btree.h index 241261dc6..96f4c4c60 100644 --- a/src/btree.h +++ b/src/btree.h @@ -317,6 +317,7 @@ struct BtreePayload { int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, int flags, int seekResult); int sqlite3BtreeFirst(BtCursor*, int *pRes); +int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes); int sqlite3BtreeLast(BtCursor*, int *pRes); int sqlite3BtreeNext(BtCursor*, int flags); int sqlite3BtreeEof(BtCursor*); diff --git a/src/build.c b/src/build.c index 5bd3aac3c..5495cef18 100644 --- a/src/build.c +++ b/src/build.c @@ -4219,7 +4219,6 @@ void sqlite3CreateIndex( assert( j<=0x7fff ); if( j<0 ){ j = pTab->iPKey; - pIndex->bIdxRowid = 1; }else{ if( pTab->aCol[j].notNull==0 ){ pIndex->uniqNotNull = 0; @@ -5138,16 +5137,22 @@ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ ** are deleted by this function. */ SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){ - assert( p1 && p1->nSrc==1 ); + assert( p1 ); + assert( p2 || pParse->nErr ); + assert( p2==0 || p2->nSrc>=1 ); + testcase( p1->nSrc==0 ); if( p2 ){ - SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, 1); + int nOld = p1->nSrc; + SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, nOld); if( pNew==0 ){ sqlite3SrcListDelete(pParse->db, p2); }else{ p1 = pNew; - memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem)); + memcpy(&p1->a[nOld], p2->a, p2->nSrc*sizeof(SrcItem)); + assert( nOld==1 || (p2->a[0].fg.jointype & JT_LTORJ)==0 ); + assert( p1->nSrc>=1 ); + p1->a[0].fg.jointype |= (JT_LTORJ & p2->a[0].fg.jointype); sqlite3DbFree(pParse->db, p2); - p1->a[0].fg.jointype |= (JT_LTORJ & p1->a[1].fg.jointype); } } return p1; diff --git a/src/expr.c b/src/expr.c index 606a4cd7e..67c97930d 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1144,7 +1144,7 @@ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ return pLeft; }else{ u32 f = pLeft->flags | pRight->flags; - if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse|EP_HasFunc))==EP_IsFalse && !IN_RENAME_OBJECT ){ sqlite3ExprDeferredDelete(pParse, pLeft); @@ -2374,6 +2374,86 @@ Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ } /* +** Return true if it might be advantageous to compute the right operand +** of expression pExpr first, before the left operand. +** +** Normally the left operand is computed before the right operand. But if +** the left operand contains a subquery and the right does not, then it +** might be more efficient to compute the right operand first. +*/ +static int exprEvalRhsFirst(Expr *pExpr){ + if( ExprHasProperty(pExpr->pLeft, EP_Subquery) + && !ExprHasProperty(pExpr->pRight, EP_Subquery) + ){ + return 1; + }else{ + return 0; + } +} + +/* +** Compute the two operands of a binary operator. +** +** If either operand contains a subquery, then the code strives to +** compute the operand containing the subquery second. If the other +** operand evalutes to NULL, then a jump is made. The address of the +** IsNull operand that does this jump is returned. The caller can use +** this to optimize the computation so as to avoid doing the potentially +** expensive subquery. +** +** If no optimization opportunities exist, return 0. +*/ +static int exprComputeOperands( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The comparison expression */ + int *pR1, /* OUT: Register holding the left operand */ + int *pR2, /* OUT: Register holding the right operand */ + int *pFree1, /* OUT: Temp register to free if not zero */ + int *pFree2 /* OUT: Another temp register to free if not zero */ +){ + int addrIsNull; + int r1, r2; + Vdbe *v = pParse->pVdbe; + + assert( v!=0 ); + /* + ** If the left operand contains a (possibly expensive) subquery and the + ** right operand does not and the right operation might be NULL, + ** then compute the right operand first and do an IsNull jump if the + ** right operand evalutes to NULL. + */ + if( exprEvalRhsFirst(pExpr) && sqlite3ExprCanBeNull(pExpr->pRight) ){ + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pFree2); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, r2); + VdbeComment((v, "skip left operand")); + VdbeCoverage(v); + }else{ + r2 = 0; /* Silence a false-positive uninit-var warning in MSVC */ + addrIsNull = 0; + } + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, pFree1); + if( addrIsNull==0 ){ + /* + ** If the right operand contains a subquery and the left operand does not + ** and the left operand might be NULL, then check the left operand do + ** an IsNull check on the left operand before computing the right + ** operand. + */ + if( ExprHasProperty(pExpr->pRight, EP_Subquery) + && sqlite3ExprCanBeNull(pExpr->pLeft) + ){ + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, r1); + VdbeComment((v, "skip right operand")); + VdbeCoverage(v); + } + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pFree2); + } + *pR1 = r1; + *pR2 = r2; + return addrIsNull; +} + +/* ** pExpr is a TK_FUNCTION node. Try to determine whether or not the ** function is a constant function. A function is constant if all of ** the following are true: @@ -3817,17 +3897,23 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ VdbeComment((v, "Init EXISTS result")); } if( pSel->pLimit ){ - /* The subquery already has a limit. If the pre-existing limit is X - ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ - sqlite3 *db = pParse->db; - pLimit = sqlite3Expr(db, TK_INTEGER, "0"); - if( pLimit ){ - pLimit->affExpr = SQLITE_AFF_NUMERIC; - pLimit = sqlite3PExpr(pParse, TK_NE, - sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); - } - sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); - pSel->pLimit->pLeft = pLimit; + /* The subquery already has a limit. If the pre-existing limit X is + ** not already integer value 1 or 0, then make the new limit X<>0 so that + ** the new limit is either 1 or 0 */ + Expr *pLeft = pSel->pLimit->pLeft; + if( ExprHasProperty(pLeft, EP_IntValue)==0 + || (pLeft->u.iValue!=1 && pLeft->u.iValue!=0) + ){ + sqlite3 *db = pParse->db; + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pLeft, 0), pLimit); + } + sqlite3ExprDeferredDelete(pParse, pLeft); + pSel->pLimit->pLeft = pLimit; + } }else{ /* If there is no pre-existing limit add a limit of 1 */ pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); @@ -3915,7 +4001,6 @@ static void sqlite3ExprCodeIN( int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ int eType; /* Type of the RHS */ int rLhs; /* Register(s) holding the LHS values */ - int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ Vdbe *v; /* Statement under construction */ int *aiMap = 0; /* Map from vector field to index column */ char *zAff = 0; /* Affinity string for comparisons */ @@ -3978,19 +4063,8 @@ static void sqlite3ExprCodeIN( ** by code generated below. */ assert( pParse->okConstFactor==okConstFactor ); pParse->okConstFactor = 0; - rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + rLhs = exprCodeVector(pParse, pLeft, &iDummy); pParse->okConstFactor = okConstFactor; - for(i=0; i<nVector && aiMap[i]==i; i++){} /* Are LHS fields reordered? */ - if( i==nVector ){ - /* LHS fields are not reordered */ - rLhs = rLhsOrig; - }else{ - /* Need to reorder the LHS fields according to aiMap */ - rLhs = sqlite3GetTempRange(pParse, nVector); - for(i=0; i<nVector; i++){ - sqlite3VdbeAddOp3(v, OP_Copy, rLhsOrig+i, rLhs+aiMap[i], 0); - } - } /* If sqlite3FindInIndex() did not find or create an index that is ** suitable for evaluating the IN operator, then evaluate using a @@ -4005,6 +4079,7 @@ static void sqlite3ExprCodeIN( int r2, regToFree; int regCkNull = 0; int ii; + assert( nVector==1 ); assert( ExprUseXList(pExpr) ); pList = pExpr->x.pList; pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); @@ -4046,6 +4121,26 @@ static void sqlite3ExprCodeIN( goto sqlite3ExprCodeIN_finished; } + if( eType!=IN_INDEX_ROWID ){ + /* If this IN operator will use an index, then the order of columns in the + ** vector might be different from the order in the index. In that case, + ** we need to reorder the LHS values to be in index order. Run Affinity + ** before reordering the columns, so that the affinity is correct. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + for(i=0; i<nVector && aiMap[i]==i; i++){} /* Are LHS fields reordered? */ + if( i!=nVector ){ + /* Need to reorder the LHS fields according to aiMap */ + int rLhsOrig = rLhs; + rLhs = sqlite3GetTempRange(pParse, nVector); + for(i=0; i<nVector; i++){ + sqlite3VdbeAddOp3(v, OP_Copy, rLhsOrig+i, rLhs+aiMap[i], 0); + } + sqlite3ReleaseTempReg(pParse, rLhsOrig); + } + } + + /* Step 2: Check to see if the LHS contains any NULL columns. If the ** LHS does contain NULLs then the result must be either FALSE or NULL. ** We will then skip the binary search of the RHS. @@ -4072,11 +4167,11 @@ static void sqlite3ExprCodeIN( /* In this case, the RHS is the ROWID of table b-tree and so we also ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 ** into a single opcode. */ + assert( nVector==1 ); sqlite3VdbeAddOp3(v, OP_SeekRowid, iTab, destIfFalse, rLhs); VdbeCoverage(v); addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ }else{ - sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); if( destIfFalse==destIfNull ){ /* Combine Step 3 and Step 5 into a single opcode */ if( ExprHasProperty(pExpr, EP_Subrtn) ){ @@ -4154,7 +4249,6 @@ static void sqlite3ExprCodeIN( sqlite3VdbeJumpHere(v, addrTruthOp); sqlite3ExprCodeIN_finished: - if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); VdbeComment((v, "end IN expr")); sqlite3ExprCodeIN_oom_error: sqlite3DbFree(pParse->db, aiMap); @@ -4269,7 +4363,12 @@ void sqlite3ExprCodeGeneratedColumn( iAddr = 0; } sqlite3ExprCodeCopy(pParse, sqlite3ColumnExpr(pTab,pCol), regOut); - if( pCol->affinity>=SQLITE_AFF_TEXT ){ + if( (pCol->colFlags & COLFLAG_VIRTUAL)!=0 + && (pTab->tabFlags & TF_Strict)!=0 + ){ + int p3 = 2+(int)(pCol - pTab->aCol); + sqlite3VdbeAddOp4(v, OP_TypeCheck, regOut, 1, p3, (char*)pTab, P4_TABLE); + }else if( pCol->affinity>=SQLITE_AFF_TEXT ){ sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); @@ -4898,6 +4997,12 @@ expr_code_doover: sqlite3VdbeLoadString(v, target, pExpr->u.zToken); return target; } + case TK_NULLS: { + /* Set a range of registers to NULL. pExpr->y.nReg registers starting + ** with target */ + sqlite3VdbeAddOp3(v, OP_Null, 0, target, target + pExpr->y.nReg - 1); + return target; + } default: { /* Make NULL the default case so that if a bug causes an illegal ** Expr node to be passed into this function, it will be handled @@ -4956,11 +5061,17 @@ expr_code_doover: case TK_NE: case TK_EQ: { Expr *pLeft = pExpr->pLeft; + int addrIsNull = 0; if( sqlite3ExprIsVector(pLeft) ){ codeVectorCompare(pParse, pExpr, target, op, p5); }else{ - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && p5!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + } sqlite3VdbeAddOp2(v, OP_Integer, 1, inReg); codeCompare(pParse, pLeft, pExpr->pRight, op, r1, r2, sqlite3VdbeCurrentAddr(v)+2, p5, @@ -4975,9 +5086,15 @@ expr_code_doover: sqlite3VdbeAddOp2(v, OP_Integer, 0, inReg); }else{ sqlite3VdbeAddOp3(v, OP_ZeroOrNull, r1, inReg, r2); + if( addrIsNull ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrIsNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, inReg); + } } testcase( regFree1==0 ); testcase( regFree2==0 ); + } break; } @@ -4993,6 +5110,7 @@ expr_code_doover: case TK_LSHIFT: case TK_RSHIFT: case TK_CONCAT: { + int addrIsNull; assert( TK_AND==OP_And ); testcase( op==TK_AND ); assert( TK_OR==OP_Or ); testcase( op==TK_OR ); assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); @@ -5004,11 +5122,23 @@ expr_code_doover: assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } sqlite3VdbeAddOp3(v, op, r2, r1, target); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrIsNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + VdbeComment((v, "short-circut value")); + } break; } case TK_UMINUS: { @@ -5583,6 +5713,25 @@ int sqlite3ExprCodeRunJustOnce( } /* +** Make arrangements to invoke OP_Null on a range of registers +** during initialization. +*/ +SQLITE_NOINLINE void sqlite3ExprNullRegisterRange( + Parse *pParse, /* Parsing context */ + int iReg, /* First register to set to NULL */ + int nReg /* Number of sequential registers to NULL out */ +){ + u8 okConstFactor = pParse->okConstFactor; + Expr t; + memset(&t, 0, sizeof(t)); + t.op = TK_NULLS; + t.y.nReg = nReg; + pParse->okConstFactor = 1; + sqlite3ExprCodeRunJustOnce(pParse, &t, iReg); + pParse->okConstFactor = okConstFactor; +} + +/* ** Generate code to evaluate an expression and store the results ** into a register. Return the register number where the results ** are stored. @@ -5857,17 +6006,27 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); if( pAlt!=pExpr ){ sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); - }else if( op==TK_AND ){ - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, - jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); }else{ - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + Expr *pFirst, *pSecond; + if( exprEvalRhsFirst(pExpr) ){ + pFirst = pExpr->pRight; + pSecond = pExpr->pLeft; + }else{ + pFirst = pExpr->pLeft; + pSecond = pExpr->pRight; + } + if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pFirst, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pSecond, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pFirst, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pSecond, dest, jumpIfNull); + } } break; } @@ -5906,10 +6065,16 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_GE: case TK_NE: case TK_EQ: { + int addrIsNull; if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && jumpIfNull!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull, ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); @@ -5924,6 +6089,13 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + if( jumpIfNull ){ + sqlite3VdbeChangeP2(v, addrIsNull, dest); + }else{ + sqlite3VdbeJumpHere(v, addrIsNull); + } + } break; } case TK_ISNULL: @@ -6031,17 +6203,27 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); if( pAlt!=pExpr ){ sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); - }else if( pExpr->op==TK_AND ){ - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); }else{ - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, - jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); + Expr *pFirst, *pSecond; + if( exprEvalRhsFirst(pExpr) ){ + pFirst = pExpr->pRight; + pSecond = pExpr->pLeft; + }else{ + pFirst = pExpr->pLeft; + pSecond = pExpr->pRight; + } + if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pFirst, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pSecond, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pFirst, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pSecond, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } } break; } @@ -6083,10 +6265,16 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_GE: case TK_NE: case TK_EQ: { + int addrIsNull; if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && jumpIfNull!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull,ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); @@ -6101,6 +6289,13 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + if( jumpIfNull ){ + sqlite3VdbeChangeP2(v, addrIsNull, dest); + }else{ + sqlite3VdbeJumpHere(v, addrIsNull); + } + } break; } case TK_ISNULL: @@ -7010,7 +7205,9 @@ static void findOrCreateAggInfoColumn( ){ struct AggInfo_col *pCol; int k; + int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + assert( mxTerm <= SMXV(i16) ); assert( pAggInfo->iFirstReg==0 ); pCol = pAggInfo->aCol; for(k=0; k<pAggInfo->nColumn; k++, pCol++){ @@ -7028,6 +7225,10 @@ static void findOrCreateAggInfoColumn( assert( pParse->db->mallocFailed ); return; } + if( k>mxTerm ){ + sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); + k = mxTerm; + } pCol = &pAggInfo->aCol[k]; assert( ExprUseYTab(pExpr) ); pCol->pTab = pExpr->y.pTab; @@ -7061,6 +7262,7 @@ fix_up_expr: if( pExpr->op==TK_COLUMN ){ pExpr->op = TK_AGG_COLUMN; } + assert( k <= SMXV(pExpr->iAgg) ); pExpr->iAgg = (i16)k; } @@ -7145,13 +7347,19 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ ** function that is already in the pAggInfo structure */ struct AggInfo_func *pItem = pAggInfo->aFunc; + int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + assert( mxTerm <= SMXV(i16) ); for(i=0; i<pAggInfo->nFunc; i++, pItem++){ if( NEVER(pItem->pFExpr==pExpr) ) break; if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ break; } } - if( i>=pAggInfo->nFunc ){ + if( i>mxTerm ){ + sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); + i = mxTerm; + assert( i<pAggInfo->nFunc ); + }else if( i>=pAggInfo->nFunc ){ /* pExpr is original. Make a new entry in pAggInfo->aFunc[] */ u8 enc = ENC(pParse->db); @@ -7205,6 +7413,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ */ assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(pExpr, EP_NoReduce); + assert( i <= SMXV(pExpr->iAgg) ); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; return WRC_Prune; diff --git a/src/os_unix.c b/src/os_unix.c index 784bc3517..7e26c8456 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1452,6 +1452,10 @@ static int findInodeInfo( storeLastErrno(pFile, errno); return SQLITE_IOERR; } + if( fsync(fd) ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_FSYNC; + } rc = osFstat(fd, &statbuf); if( rc!=0 ){ storeLastErrno(pFile, errno); diff --git a/src/os_win.c b/src/os_win.c index c7c923e77..623e30104 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3990,6 +3990,103 @@ static int winDeviceCharacteristics(sqlite3_file *id){ */ static SYSTEM_INFO winSysInfo; +/* +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function +** +** On Cygwin, 3 possible input forms are accepted: +** - If the filename starts with "<drive>:/" or "<drive>:\", +** it is converted to UTF-16 as-is. +** - If the filename contains '/', it is assumed to be a +** Cygwin absolute path, it is converted to a win32 +** absolute path in UTF-16. +** - Otherwise it must be a filename only, the win32 filename +** is returned in UTF-16. +** Note: If the function cygwin_conv_path() fails, only +** UTF-8 -> UTF-16 conversion will be done. This can only +** happen when the file path >32k, in which case winUtf8ToUnicode() +** will fail too. +*/ +static void *winConvertFromUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( osIsNT() ){ +#ifdef __CYGWIN__ + int nChar; + LPWSTR zWideFilename; + + if( osCygwin_conv_path && !(winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2])) ){ + i64 nByte; + int convertflag = CCP_POSIX_TO_WIN_W; + if( !strchr(zFilename, '/') ) convertflag |= CCP_RELATIVE; + nByte = (i64)osCygwin_conv_path(convertflag, + zFilename, 0, 0); + if( nByte>0 ){ + zConverted = sqlite3MallocZero(12+(u64)nByte); + if ( zConverted==0 ){ + return zConverted; + } + zWideFilename = zConverted; + /* Filenames should be prefixed, except when converted + * full path already starts with "\\?\". */ + if( osCygwin_conv_path(convertflag, zFilename, + zWideFilename+4, nByte)==0 ){ + if( (convertflag&CCP_RELATIVE) ){ + memmove(zWideFilename, zWideFilename+4, nByte); + }else if( memcmp(zWideFilename+4, L"\\\\", 4) ){ + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( zWideFilename[6]!='?' ){ + memmove(zWideFilename+6, zWideFilename+4, nByte); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + }else{ + memmove(zWideFilename, zWideFilename+4, nByte); + } + return zConverted; + } + sqlite3_free(zConverted); + } + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3MallocZero( nChar*sizeof(WCHAR)+12 ); + if( zWideFilename==0 ){ + return 0; + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, + zWideFilename, nChar); + if( nChar==0 ){ + sqlite3_free(zWideFilename); + zWideFilename = 0; + }else if( nChar>MAX_PATH + && winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2]) ){ + memmove(zWideFilename+4, zWideFilename, nChar*sizeof(WCHAR)); + zWideFilename[2] = '\\'; + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( nChar>MAX_PATH + && winIsDirSep(zFilename[0]) && winIsDirSep(zFilename[1]) + && zFilename[2] != '?' ){ + memmove(zWideFilename+6, zWideFilename, nChar*sizeof(WCHAR)); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + } + zConverted = zWideFilename; +#else + zConverted = winUtf8ToUnicode(zFilename); +#endif /* __CYGWIN__ */ + } +#if defined(SQLITE_WIN32_HAS_ANSI) && defined(_WIN32) + else{ + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} + #ifndef SQLITE_OMIT_WAL /* @@ -4185,103 +4282,6 @@ static int winLockSharedMemory(winShmNode *pShmNode, DWORD nMs){ /* -** Convert a UTF-8 filename into whatever form the underlying -** operating system wants filenames in. Space to hold the result -** is obtained from malloc and must be freed by the calling -** function -** -** On Cygwin, 3 possible input forms are accepted: -** - If the filename starts with "<drive>:/" or "<drive>:\", -** it is converted to UTF-16 as-is. -** - If the filename contains '/', it is assumed to be a -** Cygwin absolute path, it is converted to a win32 -** absolute path in UTF-16. -** - Otherwise it must be a filename only, the win32 filename -** is returned in UTF-16. -** Note: If the function cygwin_conv_path() fails, only -** UTF-8 -> UTF-16 conversion will be done. This can only -** happen when the file path >32k, in which case winUtf8ToUnicode() -** will fail too. -*/ -static void *winConvertFromUtf8Filename(const char *zFilename){ - void *zConverted = 0; - if( osIsNT() ){ -#ifdef __CYGWIN__ - int nChar; - LPWSTR zWideFilename; - - if( osCygwin_conv_path && !(winIsDriveLetterAndColon(zFilename) - && winIsDirSep(zFilename[2])) ){ - i64 nByte; - int convertflag = CCP_POSIX_TO_WIN_W; - if( !strchr(zFilename, '/') ) convertflag |= CCP_RELATIVE; - nByte = (i64)osCygwin_conv_path(convertflag, - zFilename, 0, 0); - if( nByte>0 ){ - zConverted = sqlite3MallocZero(12+(u64)nByte); - if ( zConverted==0 ){ - return zConverted; - } - zWideFilename = zConverted; - /* Filenames should be prefixed, except when converted - * full path already starts with "\\?\". */ - if( osCygwin_conv_path(convertflag, zFilename, - zWideFilename+4, nByte)==0 ){ - if( (convertflag&CCP_RELATIVE) ){ - memmove(zWideFilename, zWideFilename+4, nByte); - }else if( memcmp(zWideFilename+4, L"\\\\", 4) ){ - memcpy(zWideFilename, L"\\\\?\\", 8); - }else if( zWideFilename[6]!='?' ){ - memmove(zWideFilename+6, zWideFilename+4, nByte); - memcpy(zWideFilename, L"\\\\?\\UNC", 14); - }else{ - memmove(zWideFilename, zWideFilename+4, nByte); - } - return zConverted; - } - sqlite3_free(zConverted); - } - } - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); - if( nChar==0 ){ - return 0; - } - zWideFilename = sqlite3MallocZero( nChar*sizeof(WCHAR)+12 ); - if( zWideFilename==0 ){ - return 0; - } - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, - zWideFilename, nChar); - if( nChar==0 ){ - sqlite3_free(zWideFilename); - zWideFilename = 0; - }else if( nChar>MAX_PATH - && winIsDriveLetterAndColon(zFilename) - && winIsDirSep(zFilename[2]) ){ - memmove(zWideFilename+4, zWideFilename, nChar*sizeof(WCHAR)); - zWideFilename[2] = '\\'; - memcpy(zWideFilename, L"\\\\?\\", 8); - }else if( nChar>MAX_PATH - && winIsDirSep(zFilename[0]) && winIsDirSep(zFilename[1]) - && zFilename[2] != '?' ){ - memmove(zWideFilename+6, zWideFilename, nChar*sizeof(WCHAR)); - memcpy(zWideFilename, L"\\\\?\\UNC", 14); - } - zConverted = zWideFilename; -#else - zConverted = winUtf8ToUnicode(zFilename); -#endif /* __CYGWIN__ */ - } -#if defined(SQLITE_WIN32_HAS_ANSI) && defined(_WIN32) - else{ - zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); - } -#endif - /* caller will handle out of memory */ - return zConverted; -} - -/* ** This function is used to open a handle on a *-shm file. ** ** If SQLITE_ENABLE_SETLK_TIMEOUT is defined at build time, then the file diff --git a/src/parse.y b/src/parse.y index a5691cad4..617eb7303 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1429,12 +1429,21 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** expr1 IN () ** expr1 NOT IN () ** - ** simplify to constants 0 (false) and 1 (true), respectively, - ** regardless of the value of expr1. + ** simplify to constants 0 (false) and 1 (true), respectively. + ** + ** Except, do not apply this optimization if expr1 contains a function + ** because that function might be an aggregate (we don't know yet whether + ** it is or not) and if it is an aggregate, that could change the meaning + ** of the whole query. */ - sqlite3ExprUnmapAndDelete(pParse, A); - A = sqlite3Expr(pParse->db, TK_STRING, N ? "true" : "false"); - if( A ) sqlite3ExprIdToTrueFalse(A); + Expr *pB = sqlite3Expr(pParse->db, TK_STRING, N ? "true" : "false"); + if( pB ) sqlite3ExprIdToTrueFalse(pB); + if( !ExprHasProperty(A, EP_HasFunc) ){ + sqlite3ExprUnmapAndDelete(pParse, A); + A = pB; + }else{ + A = sqlite3PExpr(pParse, N ? TK_OR : TK_AND, pB, A); + } }else{ Expr *pRHS = Y->a[0].pExpr; if( Y->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && A->op!=TK_VECTOR ){ diff --git a/src/printf.c b/src/printf.c index 669ca26b0..9d95fbdd9 100644 --- a/src/printf.c +++ b/src/printf.c @@ -536,7 +536,21 @@ void sqlite3_str_vappendf( } } if( s.sign=='-' ){ - prefix = '-'; + if( flag_alternateform + && !flag_prefix + && xtype==etFLOAT + && s.iDP<=iRound + ){ + /* Suppress the minus sign if all of the following are true: + ** * The value displayed is zero + ** * The '#' flag is used + ** * The '+' flag is not used, and + ** * The format is %f + */ + prefix = 0; + }else{ + prefix = '-'; + } }else{ prefix = flag_prefix; } diff --git a/src/resolve.c b/src/resolve.c index 3961a2009..00d823940 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -926,8 +926,8 @@ static void notValidImpl( /* ** Expression p should encode a floating point value between 1.0 and 0.0. -** Return 1024 times this value. Or return -1 if p is not a floating point -** value between 1.0 and 0.0. +** Return 134,217,728 (2^27) times this value. Or return -1 if p is not +** a floating point value between 1.0 and 0.0. */ static int exprProbability(Expr *p){ double r = -1.0; @@ -1358,11 +1358,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: case TK_SELECT: - case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); #endif case TK_IN: { testcase( pExpr->op==TK_IN ); + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); if( ExprUseXSelect(pExpr) ){ int nRef = pNC->nRef; testcase( pNC->ncFlags & NC_IsCheck ); @@ -1370,6 +1372,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); assert( pExpr->x.pSelect ); + if( pExpr->op==TK_EXISTS ) pParse->bHasExists = 1; if( pNC->ncFlags & NC_SelfRef ){ notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ @@ -2280,14 +2283,17 @@ int sqlite3ResolveSelfReference( SrcList *pSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ int rc; - u8 srcSpace[SZ_SRCLIST_1]; /* Memory space for the fake SrcList */ + union { + SrcList sSrc; + u8 srcSpace[SZ_SRCLIST_1]; /* Memory space for the fake SrcList */ + } uSrc; assert( type==0 || pTab!=0 ); assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); - pSrc = (SrcList*)srcSpace; - memset(pSrc, 0, SZ_SRCLIST_1); + memset(&uSrc, 0, sizeof(uSrc)); + pSrc = &uSrc.sSrc; if( pTab ){ pSrc->nSrc = 1; pSrc->a[0].zName = pTab->zName; diff --git a/src/select.c b/src/select.c index 6c0e7c92d..db41cb493 100644 --- a/src/select.c +++ b/src/select.c @@ -384,7 +384,7 @@ static int tableAndColumnIndex( int iEnd, /* Last member of pSrc->a[] to check */ const char *zCol, /* Name of the column we are looking for */ int *piTab, /* Write index of pSrc->a[] here */ - int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int *piCol, /* Write index of pSrc->a[*piTab].pSTab->aCol[] here */ int bIgnoreHidden /* Ignore hidden columns */ ){ int i; /* For looping over tables in pSrc */ @@ -3036,7 +3036,9 @@ static int multiSelect( int priorOp; /* The SRT_ operation to apply to prior selects */ Expr *pLimit; /* Saved values of p->nLimit */ int addr; + int emptyBypass = 0; /* IfEmpty opcode to bypass RHS */ SelectDest uniondest; + testcase( p->op==TK_EXCEPT ); testcase( p->op==TK_UNION ); @@ -3075,6 +3077,8 @@ static int multiSelect( */ if( p->op==TK_EXCEPT ){ op = SRT_Except; + emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, unionTab); + VdbeCoverage(v); }else{ assert( p->op==TK_UNION ); op = SRT_Union; @@ -3095,6 +3099,7 @@ static int multiSelect( if( p->op==TK_UNION ){ p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } + if( emptyBypass ) sqlite3VdbeJumpHere(v, emptyBypass); sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->iLimit = 0; @@ -3125,9 +3130,10 @@ static int multiSelect( int tab1, tab2; int iCont, iBreak, iStart; Expr *pLimit; - int addr; + int addr, iLimit, iOffset; SelectDest intersectdest; int r1; + int emptyBypass; /* INTERSECT is different from the others since it requires ** two temporary tables. Hence it has its own case. Begin @@ -3151,15 +3157,29 @@ static int multiSelect( if( rc ){ goto multi_select_end; } + + /* Initialize LIMIT counters before checking to see if the LHS + ** is empty, in case the jump is taken */ + iBreak = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, tab1); VdbeCoverage(v); /* Code the current SELECT into temporary table "tab2" */ addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); assert( p->addrOpenEphm[1] == -1 ); p->addrOpenEphm[1] = addr; - p->pPrior = 0; + + /* Disable prior SELECTs and the LIMIT counters during the computation + ** of the RHS select */ pLimit = p->pLimit; + iLimit = p->iLimit; + iOffset = p->iOffset; + p->pPrior = 0; p->pLimit = 0; + p->iLimit = 0; + p->iOffset = 0; + intersectdest.iSDParm = tab2; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); @@ -3172,19 +3192,21 @@ static int multiSelect( p->nSelectRow = pPrior->nSelectRow; } sqlite3ExprDelete(db, p->pLimit); + + /* Reinstate the LIMIT counters prior to running the final intersect */ p->pLimit = pLimit; + p->iLimit = iLimit; + p->iOffset = iOffset; /* Generate code to take the intersection of the two temporary ** tables. */ if( rc ) break; assert( p->pEList ); - iBreak = sqlite3VdbeMakeLabel(pParse); - iCont = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_Rewind, tab1); r1 = sqlite3GetTempReg(pParse); iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); + iCont = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); @@ -3194,6 +3216,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeJumpHere(v, emptyBypass); sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); break; } @@ -3872,6 +3895,7 @@ typedef struct SubstContext { int iTable; /* Replace references to this table */ int iNewTable; /* New table number */ int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + int nSelDepth; /* Depth of sub-query recursion. Top==1 */ ExprList *pEList; /* Replacement expressions */ ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; @@ -3979,6 +4003,9 @@ static Expr *substExpr( if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){ pExpr->iTable = pSubst->iNewTable; } + if( pExpr->op==TK_AGG_FUNCTION && pExpr->op2>=pSubst->nSelDepth ){ + pExpr->op2--; + } pExpr->pLeft = substExpr(pSubst, pExpr->pLeft); pExpr->pRight = substExpr(pSubst, pExpr->pRight); if( ExprUseXSelect(pExpr) ){ @@ -4016,6 +4043,7 @@ static void substSelect( SrcItem *pItem; int i; if( !p ) return; + pSubst->nSelDepth++; do{ substExprList(pSubst, p->pEList); substExprList(pSubst, p->pGroupBy); @@ -4033,6 +4061,7 @@ static void substSelect( } } }while( doPrior && (p = p->pPrior)!=0 ); + pSubst->nSelDepth--; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -4644,7 +4673,7 @@ static int flattenSubquery( ** complete, since there may still exist Expr.pTab entries that ** refer to the subquery even after flattening. Ticket #3346. ** - ** pSubitem->pTab is always non-NULL by test restrictions and tests above. + ** pSubitem->pSTab is always non-NULL by test restrictions and tests above. */ if( ALWAYS(pSubitem->pSTab!=0) ){ Table *pTabToDel = pSubitem->pSTab; @@ -4774,6 +4803,7 @@ static int flattenSubquery( x.iTable = iParent; x.iNewTable = iNewParent; x.isOuterJoin = isOuterJoin; + x.nSelDepth = 0; x.pEList = pSub->pEList; x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); @@ -5359,6 +5389,7 @@ static int pushDownWhereTerms( x.iTable = pSrc->iCursor; x.iNewTable = pSrc->iCursor; x.isOuterJoin = 0; + x.nSelDepth = 0; x.pEList = pSubq->pEList; x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); @@ -5756,7 +5787,7 @@ With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ ** CTE expression, through routine checks to see if the reference is ** a recursive reference to the CTE. ** -** If pFrom matches a CTE according to either of these two above, pFrom->pTab +** If pFrom matches a CTE according to either of these two above, pFrom->pSTab ** and other fields are populated accordingly. ** ** Return 0 if no match is found. @@ -7384,6 +7415,84 @@ static int fromClauseTermCanBeCoroutine( } /* +** Argument pWhere is the WHERE clause belonging to SELECT statement p. This +** function attempts to transform expressions of the form: +** +** EXISTS (SELECT ...) +** +** into joins. For example, given +** +** CREATE TABLE sailors(sid INTEGER PRIMARY KEY, name TEXT); +** CREATE TABLE reserves(sid INT, day DATE, PRIMARY KEY(sid, day)); +** +** SELECT name FROM sailors AS S WHERE EXISTS ( +** SELECT * FROM reserves AS R WHERE S.sid = R.sid AND R.day = '2022-10-25' +** ); +** +** the SELECT statement may be transformed as follows: +** +** SELECT name FROM sailors AS S, reserves AS R +** WHERE S.sid = R.sid AND R.day = '2022-10-25'; +** +** **Approximately**. Really, we have to ensure that the FROM-clause term +** that was formerly inside the EXISTS is only executed once. This is handled +** by setting the SrcItem.fg.fromExists flag, which then causes code in +** the where.c file to exit the corresponding loop after the first successful +** match (if any). +*/ +static SQLITE_NOINLINE void existsToJoin( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement being optimized */ + Expr *pWhere /* part of the WHERE clause currently being examined */ +){ + if( pParse->nErr==0 + && pWhere!=0 + && !ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) + && ALWAYS(p->pSrc!=0) + && p->pSrc->nSrc<BMS + ){ + if( pWhere->op==TK_AND ){ + Expr *pRight = pWhere->pRight; + existsToJoin(pParse, p, pWhere->pLeft); + existsToJoin(pParse, p, pRight); + } + else if( pWhere->op==TK_EXISTS ){ + Select *pSub = pWhere->x.pSelect; + Expr *pSubWhere = pSub->pWhere; + if( pSub->pSrc->nSrc==1 + && (pSub->selFlags & SF_Aggregate)==0 + && !pSub->pSrc->a[0].fg.isSubquery + && pSub->pLimit==0 + ){ + memset(pWhere, 0, sizeof(*pWhere)); + pWhere->op = TK_INTEGER; + pWhere->u.iValue = 1; + ExprSetProperty(pWhere, EP_IntValue); + + assert( p->pWhere!=0 ); + pSub->pSrc->a[0].fg.fromExists = 1; + pSub->pSrc->a[0].fg.jointype |= JT_CROSS; + p->pSrc = sqlite3SrcListAppendList(pParse, p->pSrc, pSub->pSrc); + if( pSubWhere ){ + p->pWhere = sqlite3PExpr(pParse, TK_AND, p->pWhere, pSubWhere); + pSub->pWhere = 0; + } + pSub->pSrc = 0; + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSub); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x100000 ){ + TREETRACE(0x100000,pParse,p, + ("After EXISTS-to-JOIN optimization:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + existsToJoin(pParse, p, pSubWhere); + } + } + } +} + +/* ** Generate byte-code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. @@ -7751,6 +7860,13 @@ int sqlite3Select( } #endif + /* If there may be an "EXISTS (SELECT ...)" in the WHERE clause, attempt + ** to change it into a join. */ + if( pParse->bHasExists && OptimizationEnabled(db,SQLITE_ExistsToJoin) ){ + existsToJoin(pParse, p, p->pWhere); + pTabList = p->pSrc; + } + /* Do the WHERE-clause constant propagation optimization if this is ** a join. No need to spend time on this operation for non-join queries ** as the equivalent optimization will be handled by query planner in @@ -8367,6 +8483,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); VdbeComment((v, "clear abort flag")); sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); + sqlite3ExprNullRegisterRange(pParse, iAMem, pGroupBy->nExpr); /* Begin a loop that will extract all source rows in GROUP BY order. ** This might involve two separate loops with an OP_Sort in between, or @@ -8537,12 +8654,12 @@ int sqlite3Select( ** for the next GROUP BY batch. */ sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output one row")); + VdbeComment((v, "output one row of %d", p->selId)); sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); VdbeComment((v, "check abort flag")); sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - VdbeComment((v, "reset accumulator")); + VdbeComment((v, "reset accumulator %d", p->selId)); /* Update the aggregate accumulators based on the content of ** the current row @@ -8550,7 +8667,7 @@ int sqlite3Select( sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); - VdbeComment((v, "indicate data in accumulator")); + VdbeComment((v, "indicate data in accumulator %d", p->selId)); /* End of the loop */ @@ -8567,7 +8684,7 @@ int sqlite3Select( /* Output the final row of result */ sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output final row")); + VdbeComment((v, "output final row of %d", p->selId)); /* Jump over the subroutines */ @@ -8588,7 +8705,7 @@ int sqlite3Select( addrOutputRow = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); - VdbeComment((v, "Groupby result generator entry point")); + VdbeComment((v, "Groupby result generator entry point %d", p->selId)); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, pAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); @@ -8596,14 +8713,14 @@ int sqlite3Select( &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - VdbeComment((v, "end groupby result generator")); + VdbeComment((v, "end groupby result generator %d", p->selId)); /* Generate a subroutine that will reset the group-by accumulator */ sqlite3VdbeResolveLabel(v, addrReset); resetAccumulator(pParse, pAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); - VdbeComment((v, "indicate accumulator empty")); + VdbeComment((v, "indicate accumulator %d empty", p->selId)); sqlite3VdbeAddOp1(v, OP_Return, regReset); if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){ diff --git a/src/shell.c.in b/src/shell.c.in index f39823b96..3b797223a 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -106,6 +106,9 @@ typedef sqlite3_uint64 u64; typedef unsigned char u8; #include <ctype.h> #include <stdarg.h> +#ifndef _WIN32 +# include <sys/time.h> +#endif #if !defined(_WIN32) && !defined(WIN32) # include <signal.h> @@ -267,20 +270,23 @@ static int cli_strncmp(const char *a, const char *b, size_t n){ return strncmp(a,b,n); } -/* Return the current wall-clock time */ +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) +*/ static sqlite3_int64 timeOfDay(void){ - static sqlite3_vfs *clockVfs = 0; - sqlite3_int64 t; - if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); - if( clockVfs==0 ) return 0; /* Never actually happens */ - if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ - clockVfs->xCurrentTimeInt64(clockVfs, &t); - }else{ - double r; - clockVfs->xCurrentTime(clockVfs, &r); - t = (sqlite3_int64)(r*86400000.0); - } +#if defined(_WIN32) + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; return t; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif } #if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) @@ -325,8 +331,8 @@ static void endTimer(FILE *out){ sqlite3_int64 iEnd = timeOfDay(); struct rusage sEnd; getrusage(RUSAGE_SELF, &sEnd); - sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", - (iEnd - iBegin)*0.001, + sqlite3_fprintf(out, "Run Time: real %.6f user %f sys %f\n", + (iEnd - iBegin)*0.000001, timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); } @@ -404,8 +410,8 @@ static void endTimer(FILE *out){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.001, + sqlite3_fprintf(out, "Run Time: real %.6f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.000001, timeDiff(&ftUserBegin, &ftUserEnd), timeDiff(&ftKernelBegin, &ftKernelEnd)); } @@ -9328,7 +9334,8 @@ static int do_meta_command(char *zLine, ShellState *p){ " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_schema UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) " - "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " + "WHERE type!='meta' AND sql NOTNULL" + " AND name NOT LIKE 'sqlite__%' ESCAPE '_' " "ORDER BY x", callback, &data, 0 ); @@ -10804,7 +10811,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zQarg); } if( bNoSystemTabs ){ - appendText(&sSelect, "name NOT LIKE 'sqlite_%%' AND ", 0); + appendText(&sSelect, "name NOT LIKE 'sqlite__%%' ESCAPE '_' AND ", 0); } appendText(&sSelect, "sql IS NOT NULL" " ORDER BY snum, rowid", 0); @@ -11235,7 +11242,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" - " AND name NOT LIKE 'sqlite_%'" + " AND name NOT LIKE 'sqlite__%' ESCAPE '_'" " ORDER BY 1 collate nocase"; } sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); @@ -11300,7 +11307,7 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zRevText = /* Query for reversible to-blob-to-text check */ "SELECT lower(name) as tname FROM sqlite_schema\n" "WHERE type='table' AND coalesce(rootpage,0)>1\n" - "AND name NOT LIKE 'sqlite_%%'%s\n" + "AND name NOT LIKE 'sqlite__%%' ESCAPE '_'%s\n" "ORDER BY 1 collate nocase"; zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : ""); zRevText = sqlite3_mprintf( @@ -11496,7 +11503,7 @@ static int do_meta_command(char *zLine, ShellState *p){ appendText(&s, ".sqlite_schema ", 0); if( c=='t' ){ appendText(&s," WHERE type IN ('table','view')" - " AND name NOT LIKE 'sqlite_%'" + " AND name NOT LIKE 'sqlite__%' ESCAPE '_'" " AND name LIKE ?1", 0); }else{ appendText(&s," WHERE type='index'" @@ -11709,6 +11716,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { 0x08000000, 1, "OnePass" }, { 0x10000000, 1, "OrderBySubq" }, { 0x20000000, 1, "StarQuery" }, + { 0x40000000, 1, "ExistsToJoin" }, { 0xffffffff, 0, "All" }, }; unsigned int curOpt; @@ -12758,59 +12766,79 @@ static char *find_home_dir(int clearFlag){ } /* -** On non-Windows platforms, look for $XDG_CONFIG_HOME. -** If ${XDG_CONFIG_HOME}/sqlite3/sqliterc is found, return -** the path to it. If there is no $(XDG_CONFIG_HOME) then -** look for $(HOME)/.config/sqlite3/sqliterc and if found -** return that. If none of these are found, return 0. +** On non-Windows platforms, look for: +** +** - ${zEnvVar}/${zBaseName} +** - ${HOME}/${zSubdir}/${zBaseName} ** -** The string returned is obtained from sqlite3_malloc() and -** should be freed by the caller. +** $zEnvVar is intended to be the name of an XDG_... environment +** variable, e.g. XDG_CONFIG_HOME or XDG_STATE_HOME. If zEnvVar is +** NULL or getenv(zEnvVar) is NULL then fall back to the second +** option. If the selected option is not found in the filesystem, +** return 0. +** +** zSubdir may be NULL or empty, in which case ${HOME}/${zBaseName} +** becomes the fallback. +** +** Both zSubdir and zBaseName may contain subdirectory parts. zSubdir +** will conventionally be ".config" or ".local/state", which, not +** coincidentally, is the typical subdir of the corresponding XDG_... +** var with the XDG var's $HOME prefix. +** +** The returned string is obtained from sqlite3_malloc() and should be +** sqlite3_free()'d by the caller. */ -static char *find_xdg_config(void){ +static char *find_xdg_file(const char *zEnvVar, const char *zSubdir, + const char *zBaseName){ #if defined(_WIN32) || defined(WIN32) || defined(_WIN32_WCE) \ || defined(__RTP__) || defined(_WRS_KERNEL) return 0; #else - char *zConfig = 0; - const char *zXdgHome; + char *zConfigFile = 0; + const char *zXdgDir; - zXdgHome = getenv("XDG_CONFIG_HOME"); - if( zXdgHome==0 ){ - const char *zHome = getenv("HOME"); - if( zHome==0 ) return 0; - zConfig = sqlite3_mprintf("%s/.config/sqlite3/sqliterc", zHome); + zXdgDir = zEnvVar ? getenv(zEnvVar) : 0; + if( zXdgDir ){ + zConfigFile = sqlite3_mprintf("%s/%s", zXdgDir, zBaseName); }else{ - zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome); + const char * zHome = find_home_dir(0); + if( zHome==0 ) return 0; + zConfigFile = (zSubdir && *zSubdir) + ? sqlite3_mprintf("%s/%s/%s", zHome, zSubdir, zBaseName) + : sqlite3_mprintf("%s/%s", zHome, zBaseName); } - shell_check_oom(zConfig); - if( access(zConfig,0)!=0 ){ - sqlite3_free(zConfig); - zConfig = 0; + shell_check_oom(zConfigFile); + if( access(zConfigFile,0)!=0 ){ + sqlite3_free(zConfigFile); + zConfigFile = 0; } - return zConfig; + return zConfigFile; #endif } /* -** Read input from the file given by sqliterc_override. Or if that -** parameter is NULL, take input from the first of find_xdg_config() -** or ~/.sqliterc which is found. +** Read input from the file sqliterc_override. If that parameter is +** NULL, take it from find_xdg_file(), if found, or fall back to +** ~/.sqliterc. ** -** Returns the number of errors. +** Failure to read the config is only considered a failure if +** sqliterc_override is not NULL, in which case this function may emit +** a warning or, if ::bail_on_error is true, fail fatally if the file +** named by sqliterc_override is not found. */ static void process_sqliterc( ShellState *p, /* Configuration data */ const char *sqliterc_override /* Name of config file. NULL to use default */ ){ char *home_dir = NULL; - const char *sqliterc = sqliterc_override; - char *zBuf = 0; + char *sqliterc = (char*)sqliterc_override; FILE *inSaved = p->in; int savedLineno = p->lineno; if( sqliterc == NULL ){ - sqliterc = zBuf = find_xdg_config(); + sqliterc = find_xdg_file("XDG_CONFIG_HOME", + ".config", + "sqlite3/sqliterc"); } if( sqliterc == NULL ){ home_dir = find_home_dir(0); @@ -12819,11 +12847,10 @@ static void process_sqliterc( " cannot read ~/.sqliterc\n"); return; } - zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir); - shell_check_oom(zBuf); - sqliterc = zBuf; + sqliterc = sqlite3_mprintf("%s/.sqliterc",home_dir); + shell_check_oom(sqliterc); } - p->in = sqlite3_fopen(sqliterc,"rb"); + p->in = sqliterc ? sqlite3_fopen(sqliterc,"rb") : 0; if( p->in ){ if( stdin_is_interactive ){ sqlite3_fprintf(stderr,"-- Loading resources from %s\n", sqliterc); @@ -12836,7 +12863,9 @@ static void process_sqliterc( } p->in = inSaved; p->lineno = savedLineno; - sqlite3_free(zBuf); + if( sqliterc != sqliterc_override ){ + sqlite3_free(sqliterc); + } } /* @@ -13604,7 +13633,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( stdin_is_interactive ){ char *zHome; char *zHistory; - int nHistory; sqlite3_fprintf(stdout, "SQLite version %s %.19s\n" /*extra-version-info*/ "Enter \".help\" for usage hints.\n", @@ -13617,11 +13645,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } zHistory = getenv("SQLITE_HISTORY"); if( zHistory ){ - zHistory = strdup(zHistory); - }else if( (zHome = find_home_dir(0))!=0 ){ - nHistory = strlen30(zHome) + 20; - if( (zHistory = malloc(nHistory))!=0 ){ - sqlite3_snprintf(nHistory, zHistory,"%s/.sqlite_history", zHome); + zHistory = sqlite3_mprintf("%s", zHistory); + shell_check_oom(zHistory); + }else{ + zHistory = find_xdg_file("XDG_STATE_HOME", + ".local/state", + "sqlite_history"); + if( 0==zHistory && (zHome = find_home_dir(0))!=0 ){ + zHistory = sqlite3_mprintf("%s/.sqlite_history", zHome); + shell_check_oom(zHistory); } } if( zHistory ){ shell_read_history(zHistory); } @@ -13637,7 +13669,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( zHistory ){ shell_stifle_history(2000); shell_write_history(zHistory); - free(zHistory); + sqlite3_free(zHistory); } }else{ data.in = stdin; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d3164d906..65b921cc1 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -168,9 +168,9 @@ extern "C" { ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 ); ** </pre></blockquote>)^ ** -** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] -** macro. ^The sqlite3_libversion() function returns a pointer to the -** to the sqlite3_version[] string constant. The sqlite3_libversion() +** ^The sqlite3_version[] string constant contains the text of the +** [SQLITE_VERSION] macro. ^The sqlite3_libversion() function returns a +** pointer to the sqlite3_version[] string constant. The sqlite3_libversion() ** function is provided for use in DLLs since DLL users usually do not have ** direct access to string constants within the DLL. ^The ** sqlite3_libversion_number() function returns an integer equal to @@ -370,7 +370,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** without having to use a lot of C code. ** ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, -** semicolon-separate SQL statements passed into its 2nd argument, +** semicolon-separated SQL statements passed into its 2nd argument, ** in the context of the [database connection] passed in as its 1st ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row @@ -403,7 +403,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** result row is NULL then the corresponding string pointer for the ** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the ** sqlite3_exec() callback is an array of pointers to strings where each -** entry represents the name of corresponding result column as obtained +** entry represents the name of a corresponding result column as obtained ** from [sqlite3_column_name()]. ** ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer @@ -497,6 +497,9 @@ int sqlite3_exec( #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) #define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_ERROR_RESERVESIZE (SQLITE_ERROR | (4<<8)) +#define SQLITE_ERROR_KEY (SQLITE_ERROR | (5<<8)) +#define SQLITE_ERROR_UNABLE (SQLITE_ERROR | (6<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -531,6 +534,8 @@ int sqlite3_exec( #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) #define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) +#define SQLITE_IOERR_BADKEY (SQLITE_IOERR | (35<<8)) +#define SQLITE_IOERR_CODEC (SQLITE_IOERR | (36<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -589,7 +594,7 @@ int sqlite3_exec( ** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into ** [sqlite3_open_v2()] does *not* cause the underlying database file ** to be opened using O_EXCL. Passing SQLITE_OPEN_EXCLUSIVE into -** [sqlite3_open_v2()] has historically be a no-op and might become an +** [sqlite3_open_v2()] has historically been a no-op and might become an ** error in future versions of SQLite. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ @@ -683,7 +688,7 @@ int sqlite3_exec( ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods ** of an [sqlite3_io_methods] object. These values are ordered from -** lest restrictive to most restrictive. +** least restrictive to most restrictive. ** ** The argument to xLock() is always SHARED or higher. The argument to ** xUnlock is either SHARED or NONE. @@ -999,7 +1004,7 @@ struct sqlite3_io_methods { ** ** <li>[[SQLITE_FCNTL_VFSNAME]] ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of -** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** all [VFSes] in the VFS stack. The names of all VFS shims and the ** final bottom-level VFS are written into memory obtained from ** [sqlite3_malloc()] and the result is stored in the char* variable ** that the fourth parameter of [sqlite3_file_control()] points to. @@ -1013,7 +1018,7 @@ struct sqlite3_io_methods { ** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level ** [VFSes] currently in use. ^(The argument X in ** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be -** of type "[sqlite3_vfs] **". This opcodes will set *X +** of type "[sqlite3_vfs] **". This opcode will set *X ** to a pointer to the top-level VFS.)^ ** ^When there are multiple VFS shims in the stack, this opcode finds the ** upper-most shim only. @@ -1203,7 +1208,7 @@ struct sqlite3_io_methods { ** <li>[[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect ** whether or not there is a database client in another process with a wal-mode -** transaction open on the database or not. It is only available on unix.The +** transaction open on the database or not. It is only available on unix. The ** (void*) argument passed with this file-control should be a pointer to a ** value of type (int). The integer value is set to 1 if the database is a wal ** mode database and there exists at least one client in another process that @@ -1628,7 +1633,7 @@ struct sqlite3_vfs { ** SQLite interfaces so that an application usually does not need to ** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] ** calls sqlite3_initialize() so the SQLite library will be automatically -** initialized when [sqlite3_open()] is called if it has not be initialized +** initialized when [sqlite3_open()] is called if it has not been initialized ** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] ** compile-time option, then the automatic calls to sqlite3_initialize() ** are omitted and the application must call sqlite3_initialize() directly @@ -1885,21 +1890,21 @@ struct sqlite3_mem_methods { ** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation -** routines with a wrapper that simulations memory allocation failure or +** routines with a wrapper that simulates memory allocation failure or ** tracks memory usage, for example. </dd> ** ** [[SQLITE_CONFIG_SMALL_MALLOC]] <dt>SQLITE_CONFIG_SMALL_MALLOC</dt> -** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes a single argument of ** type int, interpreted as a boolean, which if true provides a hint to ** SQLite that it should avoid large memory allocations if possible. ** SQLite will run faster if it is free to make large memory allocations, -** but some application might prefer to run slower in exchange for +** but some applications might prefer to run slower in exchange for ** guarantees about memory fragmentation that are possible if large ** allocations are avoided. This hint is normally off. ** </dd> ** ** [[SQLITE_CONFIG_MEMSTATUS]] <dt>SQLITE_CONFIG_MEMSTATUS</dt> -** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes a single argument of type int, ** interpreted as a boolean, which enables or disables the collection of ** memory allocation statistics. ^(When memory allocation statistics are ** disabled, the following SQLite interfaces become non-operational: @@ -1944,7 +1949,7 @@ struct sqlite3_mem_methods { ** ^If pMem is NULL and N is non-zero, then each database connection ** does an initial bulk allocation for page cache memory ** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or -** of -1024*N bytes if N is negative, . ^If additional +** of -1024*N bytes if N is negative. ^If additional ** page cache memory is needed beyond what is provided by the initial ** allocation, then SQLite goes to [sqlite3_malloc()] separately for each ** additional cache line. </dd> @@ -1973,7 +1978,7 @@ struct sqlite3_mem_methods { ** <dd> ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. ** The argument specifies alternative low-level mutex routines to be used -** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** in place of the mutex routines built into SQLite.)^ ^SQLite makes a copy of ** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then @@ -2015,7 +2020,7 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_GETPCACHE2]] <dt>SQLITE_CONFIG_GETPCACHE2</dt> ** <dd> ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies off ** the current page cache implementation into that object.)^ </dd> ** ** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt> @@ -2032,7 +2037,7 @@ struct sqlite3_mem_methods { ** the logger function is a copy of the first parameter to the corresponding ** [sqlite3_log()] call and is intended to be a [result code] or an ** [extended result code]. ^The third parameter passed to the logger is -** log message after formatting via [sqlite3_snprintf()]. +** a log message after formatting via [sqlite3_snprintf()]. ** The SQLite logging interface is not reentrant; the logger function ** supplied by the application must not invoke any SQLite interface. ** In a multi-threaded application, the application-defined logger @@ -2223,7 +2228,7 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the second parameter to the [sqlite3_db_config()] interface. ** -** The [sqlite3_db_config()] interface is a var-args functions. It takes a +** The [sqlite3_db_config()] interface is a var-args function. It takes a ** variable number of parameters, though always at least two. The number of ** parameters passed into sqlite3_db_config() depends on which of these ** constants is given as the second parameter. This documentation page @@ -2357,8 +2362,8 @@ struct sqlite3_mem_methods { ** When the first argument to this interface is 1, then only the C-API is ** enabled and the SQL function remains disabled. If the first argument to ** this interface is 0, then both the C-API and the SQL function are disabled. -** If the first argument is -1, then no changes are made to state of either the -** C-API or the SQL function. +** If the first argument is -1, then no changes are made to the state of either +** the C-API or the SQL function. ** The second parameter is a pointer to an integer into which ** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface ** is disabled or enabled following this call. The second parameter may @@ -2476,7 +2481,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] ** <dt>SQLITE_DBCONFIG_LEGACY_ALTER_TABLE</dt> ** <dd>The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates -** the legacy behavior of the [ALTER TABLE RENAME] command such it +** the legacy behavior of the [ALTER TABLE RENAME] command such that it ** behaves as it did prior to [version 3.24.0] (2018-06-04). See the ** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for ** additional information. This feature can also be turned on and off @@ -2525,7 +2530,7 @@ struct sqlite3_mem_methods { ** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt> ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly -** created database file to have a schema format version number (the 4-byte +** created database files to have a schema format version number (the 4-byte ** integer found at offset 44 into the database header) of 1. This in turn ** means that the resulting database file will be readable and writable by ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, @@ -2552,7 +2557,7 @@ struct sqlite3_mem_methods { ** the database handle both when the SQL statement is prepared and when it ** is stepped. The flag is set (collection of statistics is enabled) ** by default. <p>This option takes two arguments: an integer and a pointer to -** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after ** processing the first argument is written into the integer that the second @@ -2595,8 +2600,8 @@ struct sqlite3_mem_methods { ** <dd>The SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE option enables or disables the ** ability of the [ATTACH DATABASE] SQL command to open a database for writing. ** This capability is enabled by default. Applications can disable or -** reenable this capability using the current DBCONFIG option. If the -** the this capability is disabled, the [ATTACH] command will still work, +** reenable this capability using the current DBCONFIG option. If +** this capability is disabled, the [ATTACH] command will still work, ** but the database will be opened read-only. If this option is disabled, ** then the ability to create a new database using [ATTACH] is also disabled, ** regardless of the value of the [SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE] @@ -2630,7 +2635,7 @@ struct sqlite3_mem_methods { ** ** <p>Most of the SQLITE_DBCONFIG options take two arguments, so that the ** overall call to [sqlite3_db_config()] has a total of four parameters. -** The first argument (the third parameter to sqlite3_db_config()) is a integer. +** The first argument (the third parameter to sqlite3_db_config()) is an integer. ** The second argument is a pointer to an integer. If the first argument is 1, ** then the option becomes enabled. If the first integer argument is 0, then the ** option is disabled. If the first argument is -1, then the option setting @@ -2920,7 +2925,7 @@ int sqlite3_is_interrupted(sqlite3*); ** ^These routines return 0 if the statement is incomplete. ^If a ** memory allocation fails, then SQLITE_NOMEM is returned. ** -** ^These routines do not parse the SQL statements thus +** ^These routines do not parse the SQL statements and thus ** will not detect syntactically incorrect SQL. ** ** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior @@ -3037,7 +3042,7 @@ int sqlite3_busy_timeout(sqlite3*, int ms); ** indefinitely if possible. The results of passing any other negative value ** are undefined. ** -** Internally, each SQLite database handle store two timeout values - the +** Internally, each SQLite database handle stores two timeout values - the ** busy-timeout (used for rollback mode databases, or if the VFS does not ** support blocking locks) and the setlk-timeout (used for blocking locks ** on wal-mode databases). The sqlite3_busy_timeout() method sets both @@ -3067,7 +3072,7 @@ int sqlite3_setlk_timeout(sqlite3*, int ms, int flags); ** This is a legacy interface that is preserved for backwards compatibility. ** Use of this interface is not recommended. ** -** Definition: A <b>result table</b> is memory data structure created by the +** Definition: A <b>result table</b> is a memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. ** @@ -3210,7 +3215,7 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is -** a no-op if is called with a NULL pointer. Passing a NULL pointer +** a no-op if it is called with a NULL pointer. Passing a NULL pointer ** to sqlite3_free() is harmless. After being freed, memory ** should neither be read nor written. Even reading previously freed ** memory might result in a segmentation fault or other severe error. @@ -3228,13 +3233,13 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_free(X). ** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation ** of at least N bytes in size or NULL if insufficient memory is available. -** ^If M is the size of the prior allocation, then min(N,M) bytes -** of the prior allocation are copied into the beginning of buffer returned +** ^If M is the size of the prior allocation, then min(N,M) bytes of the +** prior allocation are copied into the beginning of the buffer returned ** by sqlite3_realloc(X,N) and the prior allocation is freed. ** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the ** prior allocation is not freed. ** -** ^The sqlite3_realloc64(X,N) interfaces works the same as +** ^The sqlite3_realloc64(X,N) interface works the same as ** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead ** of a 32-bit signed integer. ** @@ -3284,7 +3289,7 @@ sqlite3_uint64 sqlite3_msize(void*); ** was last reset. ^The values returned by [sqlite3_memory_used()] and ** [sqlite3_memory_highwater()] include any overhead ** added by SQLite in its implementation of [sqlite3_malloc()], -** but not overhead added by the any underlying system library +** but not overhead added by any underlying system library ** routines that [sqlite3_malloc()] may call. ** ** ^The memory high-water mark is reset to the current value of @@ -3736,7 +3741,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** there is no harm in trying.) ** ** ^(<dt>[SQLITE_OPEN_SHAREDCACHE]</dt> -** <dd>The database is opened [shared cache] enabled, overriding +** <dd>The database is opened with [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** The [use of shared cache mode is discouraged] and hence shared cache @@ -3744,7 +3749,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** this option is a no-op. ** ** ^(<dt>[SQLITE_OPEN_PRIVATECACHE]</dt> -** <dd>The database is opened [shared cache] disabled, overriding +** <dd>The database is opened with [shared cache] disabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** @@ -4079,7 +4084,7 @@ sqlite3_file *sqlite3_database_file_object(const char*); ** ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of ** database filename D with corresponding journal file J and WAL file W and -** with N URI parameters key/values pairs in the array P. The result from +** an array P of N URI Key/Value pairs. The result from ** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that ** is safe to pass to routines like: ** <ul> @@ -4162,7 +4167,7 @@ void sqlite3_free_filename(sqlite3_filename); ** subsequent calls to other SQLite interface functions.)^ ** ** ^The sqlite3_errstr(E) interface returns the English-language text -** that describes the [result code] E, as UTF-8, or NULL if E is not an +** that describes the [result code] E, as UTF-8, or NULL if E is not a ** result code for which a text error message is available. ** ^(Memory to hold the error message string is managed internally ** and must not be freed by the application)^. @@ -4170,7 +4175,7 @@ void sqlite3_free_filename(sqlite3_filename); ** ^If the most recent error references a specific token in the input ** SQL, the sqlite3_error_offset() interface returns the byte offset ** of the start of that token. ^The byte offset returned by -** sqlite3_error_offset() assumes that the input SQL is UTF8. +** sqlite3_error_offset() assumes that the input SQL is UTF-8. ** ^If the most recent error does not reference a specific token in the input ** SQL, then the sqlite3_error_offset() function returns -1. ** @@ -4269,8 +4274,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** These constants define various performance limits ** that can be lowered at run-time using [sqlite3_limit()]. -** The synopsis of the meanings of the various limits is shown below. -** Additional information is available at [limits | Limits in SQLite]. +** A concise description of these limits follows, and additional information +** is available at [limits | Limits in SQLite]. ** ** <dl> ** [[SQLITE_LIMIT_LENGTH]] ^(<dt>SQLITE_LIMIT_LENGTH</dt> @@ -4335,7 +4340,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Prepare Flags ** -** These constants define various flags that can be passed into +** These constants define various flags that can be passed into the ** "prepFlags" parameter of the [sqlite3_prepare_v3()] and ** [sqlite3_prepare16_v3()] interfaces. ** @@ -4422,7 +4427,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** there is a small performance advantage to passing an nByte parameter that ** is the number of bytes in the input string <i>including</i> ** the nul-terminator. -** Note that nByte measure the length of the input in bytes, not +** Note that nByte measures the length of the input in bytes, not ** characters, even for the UTF-16 interfaces. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte @@ -4556,7 +4561,7 @@ int sqlite3_prepare16_v3( ** ** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory ** is available to hold the result, or if the result would exceed the -** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** maximum string length determined by the [SQLITE_LIMIT_LENGTH]. ** ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of ** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time @@ -4744,7 +4749,7 @@ typedef struct sqlite3_value sqlite3_value; ** ** The context in which an SQL function executes is stored in an ** sqlite3_context object. ^A pointer to an sqlite3_context object -** is always first parameter to [application-defined SQL functions]. +** is always the first parameter to [application-defined SQL functions]. ** The application-defined SQL function implementation will pass this ** pointer through into calls to [sqlite3_result_int | sqlite3_result()], ** [sqlite3_aggregate_context()], [sqlite3_user_data()], @@ -4760,7 +4765,7 @@ typedef struct sqlite3_context sqlite3_context; ** METHOD: sqlite3_stmt ** ** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, -** literals may be replaced by a [parameter] that matches one of following +** literals may be replaced by a [parameter] that matches one of the following ** templates: ** ** <ul> @@ -4805,7 +4810,7 @@ typedef struct sqlite3_context sqlite3_context; ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) -** found in first character, which is removed, or in the absence of a BOM +** found in the first character, which is removed, or in the absence of a BOM ** the byte order is the native byte order of the host ** machine for sqlite3_bind_text16() or the byte order specified in ** the 6th parameter for sqlite3_bind_text64().)^ @@ -4825,7 +4830,7 @@ typedef struct sqlite3_context sqlite3_context; ** or sqlite3_bind_text16() or sqlite3_bind_text64() then ** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL -** terminated. If any NUL characters occurs at byte offsets less than +** terminated. If any NUL characters occur at byte offsets less than ** the value of the fourth parameter then the resulting string value will ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. @@ -5037,7 +5042,7 @@ const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** METHOD: sqlite3_stmt ** ** ^These routines provide a means to determine the database, table, and -** table column that is the origin of a particular result column in +** table column that is the origin of a particular result column in a ** [SELECT] statement. ** ^The name of the database or table or column can be returned as ** either a UTF-8 or UTF-16 string. ^The _database_ routines return @@ -5481,7 +5486,7 @@ int sqlite3_column_type(sqlite3_stmt*, int iCol); ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors -** or if the statement is never been evaluated, then sqlite3_finalize() returns +** or if the statement has never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or ** [extended error code]. @@ -5606,8 +5611,8 @@ int sqlite3_reset(sqlite3_stmt *pStmt); ** ** For best security, the [SQLITE_DIRECTONLY] flag is recommended for ** all application-defined SQL functions that do not need to be -** used inside of triggers, view, CHECK constraints, or other elements of -** the database schema. This flags is especially recommended for SQL +** used inside of triggers, views, CHECK constraints, or other elements of +** the database schema. This flag is especially recommended for SQL ** functions that have side effects or reveal internal application state. ** Without this flag, an attacker might be able to modify the schema of ** a database file to include invocations of the function with parameters @@ -5638,7 +5643,7 @@ int sqlite3_reset(sqlite3_stmt *pStmt); ** [user-defined window functions|available here]. ** ** ^(If the final parameter to sqlite3_create_function_v2() or -** sqlite3_create_window_function() is not NULL, then it is destructor for +** sqlite3_create_window_function() is not NULL, then it is the destructor for ** the application data pointer. The destructor is invoked when the function ** is deleted, either by being overloaded or when the database connection ** closes.)^ ^The destructor is also invoked if the call to @@ -5713,7 +5718,7 @@ int sqlite3_create_window_function( /* ** CAPI3REF: Text Encodings ** -** These constant define integer codes that represent the various +** These constants define integer codes that represent the various ** text encodings supported by SQLite. */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ @@ -5805,7 +5810,7 @@ int sqlite3_create_window_function( ** result. ** Every function that invokes [sqlite3_result_subtype()] should have this ** property. If it does not, then the call to [sqlite3_result_subtype()] -** might become a no-op if the function is used as term in an +** might become a no-op if the function is used as a term in an ** [expression index]. On the other hand, SQL functions that never invoke ** [sqlite3_result_subtype()] should avoid setting this property, as the ** purpose of this property is to disable certain optimizations that are @@ -5932,7 +5937,7 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation ** that the xUpdate method call was invoked to implement and if -** and the prior [xColumn] method call that was invoked to extracted +** the prior [xColumn] method call that was invoked to extract ** the value for that column returned without setting a result (probably ** because it queried [sqlite3_vtab_nochange()] and found that the column ** was unchanging). ^Within an [xUpdate] method, any value for which @@ -6038,7 +6043,7 @@ unsigned int sqlite3_value_subtype(sqlite3_value*); ** METHOD: sqlite3_value ** ** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] -** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** object V and returns a pointer to that copy. ^The [sqlite3_value] returned ** is a [protected sqlite3_value] object even if the input is not. ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a ** memory allocation fails. ^If V is a [pointer value], then the result @@ -6076,7 +6081,7 @@ void sqlite3_value_free(sqlite3_value*); ** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is -** determined by the N parameter on first successful call. Changing the +** determined by the N parameter on the first successful call. Changing the ** value of N in any subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory ** allocation.)^ Within the xFinal callback, it is customary to set @@ -6238,7 +6243,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** ** Security Warning: These interfaces should not be exposed in scripting ** languages or in other circumstances where it might be possible for an -** an attacker to invoke them. Any agent that can invoke these interfaces +** attacker to invoke them. Any agent that can invoke these interfaces ** can probably also take control of the process. ** ** Database connection client data is only available for SQLite @@ -6352,7 +6357,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** pointed to by the 2nd parameter are taken as the application-defined ** function result. If the 3rd parameter is non-negative, then it ** must be the byte offset into the string where the NUL terminator would -** appear if the string where NUL terminated. If any NUL characters occur +** appear if the string were NUL terminated. If any NUL characters occur ** in the string at a byte offset that is less than the value of the 3rd ** parameter, then the resulting string will contain embedded NULs and the ** result of expressions operating on strings with embedded NULs is undefined. @@ -6410,7 +6415,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** string and preferably a string literal. The sqlite3_result_pointer() ** routine is part of the [pointer passing interface] added for SQLite 3.20.0. ** -** If these routines are called from within the different thread +** If these routines are called from within a different thread ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ @@ -6816,7 +6821,7 @@ sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** METHOD: sqlite3 ** ** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name -** for the N-th database on database connection D, or a NULL pointer of N is +** for the N-th database on database connection D, or a NULL pointer if N is ** out of range. An N value of 0 means the main database file. An N of 1 is ** the "temp" schema. Larger values of N correspond to various ATTACH-ed ** databases. @@ -6911,7 +6916,7 @@ int sqlite3_txn_state(sqlite3*,const char *zSchema); ** <dd>The SQLITE_TXN_READ state means that the database is currently ** in a read transaction. Content has been read from the database file ** but nothing in the database file has changed. The transaction state -** will advanced to SQLITE_TXN_WRITE if any changes occur and there are +** will be advanced to SQLITE_TXN_WRITE if any changes occur and there are ** no other conflicting concurrent write transactions. The transaction ** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or ** [COMMIT].</dd> @@ -6920,7 +6925,7 @@ int sqlite3_txn_state(sqlite3*,const char *zSchema); ** <dd>The SQLITE_TXN_WRITE state means that the database is currently ** in a write transaction. Content has been written to the database file ** but has not yet committed. The transaction state will change to -** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].</dd> +** SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].</dd> */ #define SQLITE_TXN_NONE 0 #define SQLITE_TXN_READ 1 @@ -7201,7 +7206,7 @@ int sqlite3_db_release_memory(sqlite3*); ** CAPI3REF: Impose A Limit On Heap Size ** ** These interfaces impose limits on the amount of heap memory that will be -** by all database connections within a single process. +** used by all database connections within a single process. ** ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the ** soft limit on the amount of heap memory that may be allocated by SQLite. @@ -7259,7 +7264,7 @@ int sqlite3_db_release_memory(sqlite3*); ** </ul>)^ ** ** The circumstances under which SQLite will enforce the heap limits may -** changes in future releases of SQLite. +** change in future releases of SQLite. */ sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); @@ -7374,8 +7379,8 @@ int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it constructs a name "sqlite3_X_init" where the -** X is consists of the lower-case equivalent of all ASCII alphabetic +** If that does not work, it constructs a name "sqlite3_X_init" where +** X consists of the lower-case equivalent of all ASCII alphabetic ** characters in the filename from the last "/" to the first following ** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns @@ -7446,7 +7451,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three ** arguments and expects an integer result as if the signature of the -** entry point where as follows: +** entry point were as follows: ** ** <blockquote><pre> ** int xEntryPoint( @@ -7610,7 +7615,7 @@ struct sqlite3_module { ** virtual table and might not be checked again by the byte code.)^ ^(The ** aConstraintUsage[].omit flag is an optimization hint. When the omit flag ** is left in its default setting of false, the constraint will always be -** checked separately in byte code. If the omit flag is change to true, then +** checked separately in byte code. If the omit flag is changed to true, then ** the constraint may or may not be checked in byte code. In other words, ** when the omit flag is true there is no guarantee that the constraint will ** not be checked again using byte code.)^ @@ -7636,7 +7641,7 @@ struct sqlite3_module { ** The xBestIndex method may optionally populate the idxFlags field with a ** mask of SQLITE_INDEX_SCAN_* flags. One such flag is ** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN] -** output to show the idxNum has hex instead of as decimal. Another flag is +** output to show the idxNum as hex instead of as decimal. Another flag is ** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will ** return at most one row. ** @@ -7777,7 +7782,7 @@ struct sqlite3_index_info { ** the implementation of the [virtual table module]. ^The fourth ** parameter is an arbitrary client data pointer that is passed through ** into the [xCreate] and [xConnect] methods of the virtual table module -** when a new virtual table is be being created or reinitialized. +** when a new virtual table is being created or reinitialized. ** ** ^The sqlite3_create_module_v2() interface has a fifth parameter which ** is a pointer to a destructor for the pClientData. ^SQLite will @@ -7942,7 +7947,7 @@ typedef struct sqlite3_blob sqlite3_blob; ** in *ppBlob. Otherwise an [error code] is returned and, unless the error ** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided ** the API is not misused, it is always safe to call [sqlite3_blob_close()] -** on *ppBlob after this function it returns. +** on *ppBlob after this function returns. ** ** This function fails with SQLITE_ERROR if any of the following are true: ** <ul> @@ -8062,7 +8067,7 @@ int sqlite3_blob_close(sqlite3_blob *); ** ** ^Returns the size in bytes of the BLOB accessible via the ** successfully opened [BLOB handle] in its only argument. ^The -** incremental blob I/O routines can only read or overwriting existing +** incremental blob I/O routines can only read or overwrite existing ** blob content; they cannot change the size of a blob. ** ** This routine only works on a [BLOB handle] which has been created @@ -8212,7 +8217,7 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() ** routine returns NULL if it is unable to allocate the requested -** mutex. The argument to sqlite3_mutex_alloc() must one of these +** mutex. The argument to sqlite3_mutex_alloc() must be one of these ** integer constants: ** ** <ul> @@ -8445,7 +8450,7 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); ** CAPI3REF: Retrieve the mutex for a database connection ** METHOD: sqlite3 ** -** ^This interface returns a pointer the [sqlite3_mutex] object that +** ^This interface returns a pointer to the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument ** when the [threading mode] is Serialized. ** ^If the [threading mode] is Single-thread or Multi-thread then this @@ -8568,7 +8573,7 @@ int sqlite3_test_control(int op, ...); ** CAPI3REF: SQL Keyword Checking ** ** These routines provide access to the set of SQL language keywords -** recognized by SQLite. Applications can uses these routines to determine +** recognized by SQLite. Applications can use these routines to determine ** whether or not a specific identifier needs to be escaped (for example, ** by enclosing in double-quotes) so as not to confuse the parser. ** @@ -8736,7 +8741,7 @@ void sqlite3_str_reset(sqlite3_str*); ** content of the dynamic string under construction in X. The value ** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X ** and might be freed or altered by any subsequent method on the same -** [sqlite3_str] object. Applications must not used the pointer returned +** [sqlite3_str] object. Applications must not use the pointer returned by ** [sqlite3_str_value(X)] after any subsequent method call on the same ** object. ^Applications may change the content of the string returned ** by [sqlite3_str_value(X)] as long as they do not write into any bytes @@ -8822,7 +8827,7 @@ int sqlite3_status64( ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they -** where too large (they were larger than the "sz" parameter to +** were too large (they were larger than the "sz" parameter to ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.</dd>)^ ** @@ -8906,28 +8911,29 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt> ** <dd>This parameter returns the number of malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.</dd>)^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE</dt> -** <dd>This parameter returns the number malloc attempts that might have +** <dd>This parameter returns the number of malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of ** memory requested being larger than the lookaside slot size. ** Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.</dd>)^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL</dt> -** <dd>This parameter returns the number malloc attempts that might have +** <dd>This parameter returns the number of malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside ** memory already being in use. ** Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.</dd>)^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(<dt>SQLITE_DBSTATUS_CACHE_USED</dt> ** <dd>This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** </dd> ** ** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] ** ^(<dt>SQLITE_DBSTATUS_CACHE_USED_SHARED</dt> @@ -8936,10 +8942,10 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** memory used by that pager cache is divided evenly between the attached ** connections.)^ In other words, if none of the pager caches associated ** with the database connection are shared, this request returns the same -** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** value as DBSTATUS_CACHE_USED. Or, if one or more of the pager caches are ** shared, the value returned by this call will be smaller than that returned ** by DBSTATUS_CACHE_USED. ^The highwater mark associated with -** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0.</dd> ** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(<dt>SQLITE_DBSTATUS_SCHEMA_USED</dt> ** <dd>This parameter returns the approximate number of bytes of heap @@ -8949,6 +8955,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** schema memory is shared with other database connections due to ** [shared cache mode] being enabled. ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** </dd> ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(<dt>SQLITE_DBSTATUS_STMT_USED</dt> ** <dd>This parameter returns the approximate number of bytes of heap @@ -8985,7 +8992,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** been written to disk in the middle of a transaction due to the page ** cache overflowing. Transactions are more efficient if they are written ** to disk all at once. When pages spill mid-transaction, that introduces -** additional overhead. This parameter can be used help identify +** additional overhead. This parameter can be used to help identify ** inefficiencies that can be resolved by increasing the cache size. ** </dd> ** @@ -9056,13 +9063,13 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** [[SQLITE_STMTSTATUS_SORT]] <dt>SQLITE_STMTSTATUS_SORT</dt> ** <dd>^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to -** improvement performance through careful use of indices.</dd> +** improve performance through careful use of indices.</dd> ** ** [[SQLITE_STMTSTATUS_AUTOINDEX]] <dt>SQLITE_STMTSTATUS_AUTOINDEX</dt> ** <dd>^This is the number of rows inserted into transient indices that ** were created automatically in order to help joins run faster. ** A non-zero value in this counter may indicate an opportunity to -** improvement performance by adding permanent indices that do not +** improve performance by adding permanent indices that do not ** need to be reinitialized each time the statement is run.</dd> ** ** [[SQLITE_STMTSTATUS_VM_STEP]] <dt>SQLITE_STMTSTATUS_VM_STEP</dt> @@ -9071,19 +9078,19 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** to 2147483647. The number of virtual machine operations can be ** used as a proxy for the total work done by the prepared statement. ** If the number of virtual machine operations exceeds 2147483647 -** then the value returned by this statement status code is undefined. +** then the value returned by this statement status code is undefined.</dd> ** ** [[SQLITE_STMTSTATUS_REPREPARE]] <dt>SQLITE_STMTSTATUS_REPREPARE</dt> ** <dd>^This is the number of times that the prepare statement has been ** automatically regenerated due to schema changes or changes to -** [bound parameters] that might affect the query plan. +** [bound parameters] that might affect the query plan.</dd> ** ** [[SQLITE_STMTSTATUS_RUN]] <dt>SQLITE_STMTSTATUS_RUN</dt> ** <dd>^This is the number of times that the prepared statement has ** been run. A single "run" for the purposes of this counter is one ** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()]. ** The counter is incremented on the first [sqlite3_step()] call of each -** cycle. +** cycle.</dd> ** ** [[SQLITE_STMTSTATUS_FILTER_MISS]] ** [[SQLITE_STMTSTATUS_FILTER HIT]] @@ -9093,7 +9100,7 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** step was bypassed because a Bloom filter returned not-found. The ** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of ** times that the Bloom filter returned a find, and thus the join step -** had to be processed as normal. +** had to be processed as normal.</dd> ** ** [[SQLITE_STMTSTATUS_MEMUSED]] <dt>SQLITE_STMTSTATUS_MEMUSED</dt> ** <dd>^This is the approximate number of bytes of heap memory @@ -9198,9 +9205,9 @@ struct sqlite3_pcache_page { ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must -** be allocated by the cache. ^szPage will always a power of two. ^The +** be allocated by the cache. ^szPage will always be a power of two. ^The ** second parameter szExtra is a number of bytes of extra storage -** associated with each page cache entry. ^The szExtra parameter will +** associated with each page cache entry. ^The szExtra parameter will be ** a number less than 250. SQLite will use the ** extra szExtra bytes on each page to store metadata about the underlying ** database page on disk. The value passed into szExtra depends @@ -9208,17 +9215,17 @@ struct sqlite3_pcache_page { ** ^The third argument to xCreate(), bPurgeable, is true if the cache being ** created will be used to cache database pages of a file stored on disk, or ** false if it is used for an in-memory database. The cache implementation -** does not have to do anything special based with the value of bPurgeable; +** does not have to do anything special based upon the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. ** ^In other words, calls to xUnpin() on a cache with bPurgeable set to ** false will always have the "discard" flag set to true. -** ^Hence, a cache created with bPurgeable false will +** ^Hence, a cache created with bPurgeable set to false will ** never contain any unpinned pages. ** ** [[the xCachesize() page cache method]] ** ^(The xCachesize() method may be called at any time by SQLite to set the -** suggested maximum cache-size (number of pages stored by) the cache +** suggested maximum cache-size (number of pages stored) for the cache ** instance passed as the first argument. This is the value configured using ** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this @@ -9245,12 +9252,12 @@ struct sqlite3_pcache_page { ** implementation must return a pointer to the page buffer with its content ** intact. If the requested page is not already in the cache, then the ** cache implementation should use the value of the createFlag -** parameter to help it determined what action to take: +** parameter to help it determine what action to take: ** ** <table border=1 width=85% align=center> ** <tr><th> createFlag <th> Behavior when page is not already in cache ** <tr><td> 0 <td> Do not allocate a new page. Return NULL. -** <tr><td> 1 <td> Allocate a new page if it easy and convenient to do so. +** <tr><td> 1 <td> Allocate a new page if it is easy and convenient to do so. ** Otherwise return NULL. ** <tr><td> 2 <td> Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. @@ -9267,7 +9274,7 @@ struct sqlite3_pcache_page { ** as its second argument. If the third parameter, discard, is non-zero, ** then the page must be evicted from the cache. ** ^If the discard parameter is -** zero, then the page may be discarded or retained at the discretion of +** zero, then the page may be discarded or retained at the discretion of the ** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** @@ -9285,7 +9292,7 @@ struct sqlite3_pcache_page { ** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal ** to the value of the iLimit parameter passed to xTruncate(). If any -** of these pages are pinned, they are implicitly unpinned, meaning that +** of these pages are pinned, they become implicitly unpinned, meaning that ** they can be safely discarded. ** ** [[the xDestroy() page cache method]] @@ -9465,7 +9472,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** external process or via a database connection other than the one being ** used by the backup operation, then the backup will be automatically ** restarted by the next call to sqlite3_backup_step(). ^If the source -** database is modified by the using the same database connection as is used +** database is modified by using the same database connection as is used ** by the backup operation, then the backup database is automatically ** updated at the same time. ** @@ -9482,7 +9489,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** and may not be used following a call to sqlite3_backup_finish(). ** ** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no -** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() errors occurred, regardless of whether or not ** sqlite3_backup_step() completed. ** ^If an out-of-memory condition or IO error occurred during any prior ** sqlite3_backup_step() call on the same [sqlite3_backup] object, then @@ -9584,7 +9591,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** application receives an SQLITE_LOCKED error, it may call the ** sqlite3_unlock_notify() method with the blocked connection handle as ** the first argument to register for a callback that will be invoked -** when the blocking connections current transaction is concluded. ^The +** when the blocking connection's current transaction is concluded. ^The ** callback is invoked from within the [sqlite3_step] or [sqlite3_close] ** call that concludes the blocking connection's transaction. ** @@ -9604,7 +9611,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connection's ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -10002,7 +10009,7 @@ int sqlite3_vtab_config(sqlite3*, int op, ...); ** support constraints. In this configuration (which is the default) if ** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire ** statement is rolled back as if [ON CONFLICT | OR ABORT] had been -** specified as part of the users SQL statement, regardless of the actual +** specified as part of the user's SQL statement, regardless of the actual ** ON CONFLICT mode specified. ** ** If X is non-zero, then the virtual table implementation guarantees @@ -10036,7 +10043,7 @@ int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]<dt>SQLITE_VTAB_INNOCUOUS</dt> ** <dd>Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a @@ -10204,7 +10211,7 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** </table> ** ** ^For the purposes of comparing virtual table output values to see if the -** values are same value for sorting purposes, two NULL values are considered +** values are the same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** @@ -10214,7 +10221,7 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** ** ^A virtual table implementation is always free to return rows in any order ** it wants, as long as the "orderByConsumed" flag is not set. ^When the -** the "orderByConsumed" flag is unset, the query planner will add extra +** "orderByConsumed" flag is unset, the query planner will add extra ** [bytecode] to ensure that the final results returned by the SQL query are ** ordered correctly. The use of the "orderByConsumed" flag and the ** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful @@ -10311,7 +10318,7 @@ int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); ** sqlite3_vtab_in_next(X,P) should be one of the parameters to the ** xFilter method which invokes these routines, and specifically ** a parameter that was previously selected for all-at-once IN constraint -** processing use the [sqlite3_vtab_in()] interface in the +** processing using the [sqlite3_vtab_in()] interface in the ** [xBestIndex|xBestIndex method]. ^(If the X parameter is not ** an xFilter argument that was selected for all-at-once IN constraint ** processing, then these routines return [SQLITE_ERROR].)^ @@ -10366,7 +10373,7 @@ int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); ** and only if *V is set to a value. ^The sqlite3_vtab_rhs_value(P,J,V) ** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th ** constraint is not available. ^The sqlite3_vtab_rhs_value() interface -** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if +** can return a result code other than SQLITE_OK or SQLITE_NOTFOUND if ** something goes wrong. ** ** The sqlite3_vtab_rhs_value() interface is usually only successful if @@ -10394,8 +10401,8 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** KEYWORDS: {conflict resolution mode} ** ** These constants are returned by [sqlite3_vtab_on_conflict()] to -** inform a [virtual table] implementation what the [ON CONFLICT] mode -** is for the SQL statement being evaluated. +** inform a [virtual table] implementation of the [ON CONFLICT] mode +** for the SQL statement being evaluated. ** ** Note that the [SQLITE_IGNORE] constant is also used as a potential ** return value from the [sqlite3_set_authorizer()] callback and that @@ -10435,39 +10442,39 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** [[SQLITE_SCANSTAT_EST]] <dt>SQLITE_SCANSTAT_EST</dt> ** <dd>^The "double" variable pointed to by the V parameter will be set to the ** query planner's estimate for the average number of rows output from each -** iteration of the X-th loop. If the query planner's estimates was accurate, +** iteration of the X-th loop. If the query planner's estimate was accurate, ** then this value will approximate the quotient NVISIT/NLOOP and the ** product of this value for all prior loops with the same SELECTID will -** be the NLOOP value for the current loop. +** be the NLOOP value for the current loop.</dd> ** ** [[SQLITE_SCANSTAT_NAME]] <dt>SQLITE_SCANSTAT_NAME</dt> ** <dd>^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the name of the index or table -** used for the X-th loop. +** used for the X-th loop.</dd> ** ** [[SQLITE_SCANSTAT_EXPLAIN]] <dt>SQLITE_SCANSTAT_EXPLAIN</dt> ** <dd>^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] -** description for the X-th loop. +** description for the X-th loop.</dd> ** ** [[SQLITE_SCANSTAT_SELECTID]] <dt>SQLITE_SCANSTAT_SELECTID</dt> ** <dd>^The "int" variable pointed to by the V parameter will be set to the ** id for the X-th query plan element. The id value is unique within the ** statement. The select-id is the same value as is output in the first -** column of an [EXPLAIN QUERY PLAN] query. +** column of an [EXPLAIN QUERY PLAN] query.</dd> ** ** [[SQLITE_SCANSTAT_PARENTID]] <dt>SQLITE_SCANSTAT_PARENTID</dt> ** <dd>The "int" variable pointed to by the V parameter will be set to the -** the id of the parent of the current query element, if applicable, or +** id of the parent of the current query element, if applicable, or ** to zero if the query element has no parent. This is the same value as -** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** returned in the second column of an [EXPLAIN QUERY PLAN] query.</dd> ** ** [[SQLITE_SCANSTAT_NCYCLE]] <dt>SQLITE_SCANSTAT_NCYCLE</dt> ** <dd>The sqlite3_int64 output value is set to the number of cycles, ** according to the processor time-stamp counter, that elapsed while the ** query element was being processed. This value is not available for ** all query elements - if it is unavailable the output variable is -** set to -1. +** set to -1.</dd> ** </dl> */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -10508,8 +10515,8 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. ** ** Parameter "idx" identifies the specific query element to retrieve statistics -** for. Query elements are numbered starting from zero. A value of -1 may be -** to query for statistics regarding the entire query. ^If idx is out of range +** for. Query elements are numbered starting from zero. A value of -1 may +** retrieve statistics for the entire query. ^If idx is out of range ** - less than -1 or greater than or equal to the total number of query ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. @@ -10521,14 +10528,14 @@ int sqlite3_stmt_scanstatus( int idx, /* Index of loop to report on */ int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ void *pOut /* Result written here */ -); +); int sqlite3_stmt_scanstatus_v2( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ int idx, /* Index of loop to report on */ int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ int flags, /* Mask of flags defined below */ void *pOut /* Result written here */ -); +); /* ** CAPI3REF: Prepared Statement Scan Status @@ -10552,7 +10559,7 @@ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); ** METHOD: sqlite3 ** ** ^If a write-transaction is open on [database connection] D when the -** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** [sqlite3_db_cacheflush(D)] interface is invoked, any dirty ** pages in the pager-cache that are not currently in use are written out ** to disk. A dirty page may be in use if a database cursor created by an ** active SQL statement is reading from it, or if it is page 1 of a database @@ -10666,8 +10673,8 @@ int sqlite3_db_cacheflush(sqlite3*); ** triggers; and so forth. ** ** When the [sqlite3_blob_write()] API is used to update a blob column, -** the pre-update hook is invoked with SQLITE_DELETE. This is because the -** in this case the new values are not available. In this case, when a +** the pre-update hook is invoked with SQLITE_DELETE, because +** the new values are not yet available. In this case, when a ** callback made with op==SQLITE_DELETE is actually a write using the ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns ** the index of the column being written. In other cases, where the @@ -10785,7 +10792,7 @@ typedef struct sqlite3_snapshot { ** The [sqlite3_snapshot_get()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( +int sqlite3_snapshot_get( sqlite3 *db, const char *zSchema, sqlite3_snapshot **ppSnapshot @@ -10834,7 +10841,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** The [sqlite3_snapshot_open()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( +int sqlite3_snapshot_open( sqlite3 *db, const char *zSchema, sqlite3_snapshot *pSnapshot @@ -10851,7 +10858,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( ** The [sqlite3_snapshot_free()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); +void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. @@ -10878,7 +10885,7 @@ SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( +int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, sqlite3_snapshot *p2 ); @@ -10906,7 +10913,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); +int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Serialize a database @@ -10920,7 +10927,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); ** For an ordinary on-disk database file, the serialization is just a ** copy of the disk file. For an in-memory database or a "TEMP" database, ** the serialization is the same sequence of bytes which would be written -** to disk if that database where backed up to disk. +** to disk if that database were backed up to disk. ** ** The usual case is that sqlite3_serialize() copies the serialization of ** the database into memory obtained from [sqlite3_malloc64()] and returns @@ -10929,7 +10936,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); ** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations ** are made, and the sqlite3_serialize() function will return a pointer ** to the contiguous memory representation of the database that SQLite -** is currently using for that database, or NULL if the no such contiguous +** is currently using for that database, or NULL if no such contiguous ** memory representation of the database exists. A contiguous memory ** representation of the database will usually only exist if there has ** been a prior call to [sqlite3_deserialize(D,S,...)] with the same @@ -10978,14 +10985,15 @@ unsigned char *sqlite3_serialize( /* ** CAPI3REF: Deserialize a database ** -** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the -** [database connection] D to disconnect from database S and then -** reopen S as an in-memory database based on the serialization contained -** in P. The serialized database P is N bytes in size. M is the size of -** the buffer P, which might be larger than N. If M is larger than N, and -** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is -** permitted to add content to the in-memory database as long as the total -** size does not exceed M bytes. +** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the [database +** connection] D to disconnect from database S and then reopen S as an +** in-memory database based on the serialization contained in P. If S +** is a NULL pointer, the main database is used. The serialized +** database P is N bytes in size. M is the size of the buffer P, +** which might be larger than N. If M is larger than N, and the +** SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is +** permitted to add content to the in-memory database as long as the +** total size does not exceed M bytes. ** ** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will ** invoke sqlite3_free() on the serialization buffer when the database @@ -11000,7 +11008,7 @@ unsigned char *sqlite3_serialize( ** database is currently in a read transaction or is involved in a backup ** operation. ** -** It is not possible to deserialized into the TEMP database. If the +** It is not possible to deserialize into the TEMP database. If the ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the ** function returns SQLITE_ERROR. ** @@ -11022,7 +11030,7 @@ int sqlite3_deserialize( sqlite3 *db, /* The database connection */ const char *zSchema, /* Which DB to reopen with the deserialization */ unsigned char *pData, /* The serialized database content */ - sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szDb, /* Number of bytes in the deserialization */ sqlite3_int64 szBuf, /* Total size of buffer pData[] */ unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ ); @@ -11030,7 +11038,7 @@ int sqlite3_deserialize( /* ** CAPI3REF: Flags for sqlite3_deserialize() ** -** The following are allowed values for 6th argument (the F argument) to +** The following are allowed values for the 6th argument (the F argument) to ** the [sqlite3_deserialize(D,S,P,N,M,F)] interface. ** ** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ce1b77bfe..a09c94ae6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1031,8 +1031,8 @@ typedef INT16_TYPE LogEst; ** assuming n is a signed integer type. UMXV(n) is similar for unsigned ** integer types. */ -#define SMXV(n) ((((i64)1)<<(sizeof(n)-1))-1) -#define UMXV(n) ((((i64)1)<<(sizeof(n)))-1) +#define SMXV(n) ((((i64)1)<<(sizeof(n)*8-1))-1) +#define UMXV(n) ((((i64)1)<<(sizeof(n)*8))-1) /* ** Round up a number to the next larger multiple of 8. This is used @@ -1154,6 +1154,7 @@ extern u32 sqlite3TreeTrace; ** 0x00040000 SELECT tree dump after all code has been generated ** 0x00080000 NOT NULL strength reduction ** 0x00100000 Pointers are all shown as zero +** 0x00200000 EXISTS-to-JOIN optimization */ /* @@ -1926,6 +1927,7 @@ struct sqlite3 { #define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ #define SQLITE_OrderBySubq 0x10000000 /* ORDER BY in subquery helps outer */ #define SQLITE_StarQuery 0x20000000 /* Heurists for star queries */ +#define SQLITE_ExistsToJoin 0x40000000 /* The EXISTS-to-JOIN optimization */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -2164,7 +2166,7 @@ struct FuncDestructor { #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_BUILTIN|\ SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, 0, 0, #zName, } + pArg, 0, xFunc, 0, 0, 0, #zName, {0} } #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } @@ -2808,7 +2810,6 @@ struct Index { unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ - unsigned bIdxRowid:1; /* One or more of the index keys is the ROWID */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ unsigned bHasExpr:1; /* Index contains an expression, either a literal ** expression, or a reference to a VIRTUAL column */ @@ -2896,7 +2897,7 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ - u16 nSortingColumn; /* Number of columns in the sorting index */ + u32 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int iFirstReg; /* First register in range for aCol[] and aFunc[] */ @@ -2905,8 +2906,8 @@ struct AggInfo { Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - i16 iColumn; /* Column number within the source table */ - i16 iSorterColumn; /* Column number in the sorting index */ + int iColumn; /* Column number within the source table */ + int iSorterColumn; /* Column number in the sorting index */ } *aCol; int nColumn; /* Number of used entries in aCol[] */ int nAccumulator; /* Number of columns that show through to the output. @@ -3081,6 +3082,7 @@ struct Expr { Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL ** for a column of an index on an expression */ Window *pWin; /* EP_WinFunc: Window/Filter defn for a function */ + int nReg; /* TK_NULLS: Number of registers to NULL out */ struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ int iAddr; /* Subroutine entry address */ int regReturn; /* Register used to hold return address */ @@ -3370,6 +3372,7 @@ struct SrcItem { unsigned rowidUsed :1; /* The ROWID of this table is referenced */ unsigned fixedSchema :1; /* Uses u4.pSchema, not u4.zDatabase */ unsigned hadSchema :1; /* Had u4.zDatabase before u4.pSchema */ + unsigned fromExists :1; /* Comes from WHERE EXISTS(...) */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ @@ -3900,6 +3903,7 @@ struct Parse { u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasExists; /* Has a correlated "EXISTS (SELECT ....)" expression */ u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 bReturning; /* Coding a RETURNING trigger */ @@ -5116,6 +5120,7 @@ void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int); void sqlite3ExprCodeCopy(Parse*, Expr*, int); void sqlite3ExprCodeFactorable(Parse*, Expr*, int); int sqlite3ExprCodeRunJustOnce(Parse*, Expr*, int); +void sqlite3ExprNullRegisterRange(Parse*, int, int); int sqlite3ExprCodeTemp(Parse*, Expr*, int*); int sqlite3ExprCodeTarget(Parse*, Expr*, int); int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index ec774889b..6b6bb7167 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -196,7 +196,7 @@ ** Maximum number of pages in one database file. ** ** This is really just the default value for the max_page_count pragma. -** This value can be lowered (or raised) at run-time using that the +** This value can be lowered (or raised) at run-time using the ** max_page_count macro. */ #ifndef SQLITE_MAX_PAGE_COUNT diff --git a/src/tokenize.c b/src/tokenize.c index e4d9f5371..6f7bab35b 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -199,7 +199,7 @@ static int getToken(const unsigned char **pz){ int t; /* Token type to return */ do { z += sqlite3GetToken(z, &t); - }while( t==TK_SPACE ); + }while( t==TK_SPACE || t==TK_COMMENT ); if( t==TK_ID || t==TK_STRING || t==TK_JOIN_KW diff --git a/src/trigger.c b/src/trigger.c index 779da5e5f..799fbe57f 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -1040,7 +1040,10 @@ static void codeReturningTrigger( Returning *pReturning; Select sSelect; SrcList *pFrom; - u8 fromSpace[SZ_SRCLIST_1]; + union { + SrcList sSrc; + u8 fromSpace[SZ_SRCLIST_1]; + } uSrc; assert( v!=0 ); if( !pParse->bReturning ){ @@ -1056,8 +1059,8 @@ static void codeReturningTrigger( return; } memset(&sSelect, 0, sizeof(sSelect)); - pFrom = (SrcList*)fromSpace; - memset(pFrom, 0, SZ_SRCLIST_1); + memset(&uSrc, 0, sizeof(uSrc)); + pFrom = &uSrc.sSrc; sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); sSelect.pSrc = pFrom; pFrom->nSrc = 1; diff --git a/src/vdbe.c b/src/vdbe.c index b23bd38d2..0465ba27a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -792,6 +792,36 @@ static SQLITE_NOINLINE int vdbeColumnFromOverflow( return rc; } +/* +** Send a "statement aborts" message to the error log. +*/ +static SQLITE_NOINLINE void sqlite3VdbeLogAbort( + Vdbe *p, /* The statement that is running at the time of failure */ + int rc, /* Error code */ + Op *pOp, /* Opcode that filed */ + Op *aOp /* All opcodes */ +){ + const char *zSql = p->zSql; /* Original SQL text */ + const char *zPrefix = ""; /* Prefix added to SQL text */ + int pc; /* Opcode address */ + char zXtra[100]; /* Buffer space to store zPrefix */ + + if( p->pFrame ){ + assert( aOp[0].opcode==OP_Init ); + if( aOp[0].p4.z!=0 ){ + assert( aOp[0].p4.z[0]=='-' + && aOp[0].p4.z[1]=='-' + && aOp[0].p4.z[2]==' ' ); + sqlite3_snprintf(sizeof(zXtra), zXtra,"/* %s */ ",aOp[0].p4.z+3); + zPrefix = zXtra; + }else{ + zPrefix = "/* unknown trigger */ "; + } + } + pc = (int)(pOp - aOp); + sqlite3_log(rc, "statement aborts at %d: %s; [%s%s]", + pc, p->zErrMsg, zPrefix, zSql); +} /* ** Return the symbolic name for the data type of a pMem @@ -1317,8 +1347,7 @@ case OP_Halt: { }else{ sqlite3VdbeError(p, "%s", pOp->p4.z); } - pcx = (int)(pOp - aOp); - sqlite3_log(pOp->p1, "abort at %d: %s; [%s]", pcx, p->zErrMsg, p->zSql); + sqlite3VdbeLogAbort(p, pOp->p1, pOp, aOp); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); @@ -1697,7 +1726,7 @@ case OP_IntCopy: { /* out2 */ ** RETURNING clause. */ case OP_FkCheck: { - if( (rc = sqlite3VdbeCheckFk(p,0))!=SQLITE_OK ){ + if( (rc = sqlite3VdbeCheckFkImmediate(p))!=SQLITE_OK ){ goto abort_due_to_error; } break; @@ -3239,6 +3268,15 @@ op_column_corrupt: ** Take the affinities from the Table object in P4. If any value ** cannot be coerced into the correct type, then raise an error. ** +** If P3==0, then omit checking of VIRTUAL columns. +** +** If P3==1, then omit checking of all generated column, both VIRTUAL +** and STORED. +** +** If P3>=2, then only check column number P3-2 in the table (which will +** be a VIRTUAL column) against the value in reg[P1]. In this case, +** P2 will be 1. +** ** This opcode is similar to OP_Affinity except that this opcode ** forces the register type to the Table column type. This is used ** to implement "strict affinity". @@ -3252,8 +3290,8 @@ op_column_corrupt: ** ** <ul> ** <li> P2 should be the number of non-virtual columns in the -** table of P4. -** <li> Table P4 should be a STRICT table. +** table of P4 unless P3>1, in which case P2 will be 1. +** <li> Table P4 is a STRICT table. ** </ul> ** ** If any precondition is false, an assertion fault occurs. @@ -3262,16 +3300,28 @@ case OP_TypeCheck: { Table *pTab; Column *aCol; int i; + int nCol; assert( pOp->p4type==P4_TABLE ); pTab = pOp->p4.pTab; assert( pTab->tabFlags & TF_Strict ); - assert( pTab->nNVCol==pOp->p2 ); + assert( pOp->p3>=0 && pOp->p3<pTab->nCol+2 ); aCol = pTab->aCol; pIn1 = &aMem[pOp->p1]; - for(i=0; i<pTab->nCol; i++){ - if( aCol[i].colFlags & COLFLAG_GENERATED ){ - if( aCol[i].colFlags & COLFLAG_VIRTUAL ) continue; + if( pOp->p3<2 ){ + assert( pTab->nNVCol==pOp->p2 ); + i = 0; + nCol = pTab->nCol; + }else{ + i = pOp->p3-2; + nCol = i+1; + assert( i<pTab->nCol ); + assert( aCol[i].colFlags & COLFLAG_VIRTUAL ); + assert( pOp->p2==1 ); + } + for(; i<nCol; i++){ + if( (aCol[i].colFlags & COLFLAG_GENERATED)!=0 && pOp->p3<2 ){ + if( (aCol[i].colFlags & COLFLAG_VIRTUAL)!=0 ) continue; if( pOp->p3 ){ pIn1++; continue; } } assert( pIn1 < &aMem[pOp->p1+pOp->p2] ); @@ -3860,7 +3910,7 @@ case OP_Savepoint: { */ int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; if( isTransaction && p1==SAVEPOINT_RELEASE ){ - if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){ goto vdbe_return; } db->autoCommit = 1; @@ -3978,7 +4028,7 @@ case OP_AutoCommit: { "SQL statements in progress"); rc = SQLITE_BUSY; goto abort_due_to_error; - }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + }else if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; @@ -6348,6 +6398,32 @@ case OP_Rewind: { /* jump0, ncycle */ break; } +/* Opcode: IfEmpty P1 P2 * * * +** Synopsis: if( empty(P1) ) goto P2 +** +** Check to see if the b-tree table that cursor P1 references is empty +** and jump to P2 if it is. +*/ +case OP_IfEmpty: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + + assert( pOp->p1>=0 && pOp->p1<p->nCursor ); + assert( pOp->p2>=0 && pOp->p2<p->nOp ); + + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeIsEmpty(pCrsr, &res); + if( rc ) goto abort_due_to_error; + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + /* Opcode: Next P1 P2 P3 * P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its @@ -8219,7 +8295,14 @@ case OP_VOpen: { /* ncycle */ const sqlite3_module *pModule; assert( p->bIsReader ); - pCur = 0; + pCur = p->apCsr[pOp->p1]; + if( pCur!=0 + && ALWAYS( pCur->eCurType==CURTYPE_VTAB ) + && ALWAYS( pCur->uc.pVCur->pVtab==pOp->p4.pVtab->pVtab ) + ){ + /* This opcode is a no-op if the cursor is already open */ + break; + } pVCur = 0; pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ @@ -9161,8 +9244,7 @@ abort_due_to_error: p->rc = rc; sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(rc, "statement aborts at %d: %s; [%s]", - (int)(pOp - aOp), p->zErrMsg, p->zSql); + sqlite3VdbeLogAbort(p, rc, pOp, aOp); if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); if( rc==SQLITE_CORRUPT && db->autoCommit==0 ){ diff --git a/src/vdbe.h b/src/vdbe.h index a7aedfbb0..9a9cf3dec 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -314,7 +314,9 @@ int sqlite3VdbeHasSubProgram(Vdbe*); void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); +#ifndef SQLITE_OMIT_DATETIME_FUNCS int sqlite3NotPureFunc(sqlite3_context*); +#endif #ifdef SQLITE_ENABLE_BYTECODE_VTAB int sqlite3VdbeBytecodeVtabInit(sqlite3*); #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 0faa32747..ca28b075a 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -288,7 +288,7 @@ struct sqlite3_value { ** MEM_Int, MEM_Real, and MEM_IntReal. ** ** * MEM_Blob|MEM_Zero A blob in Mem.z of length Mem.n plus -** MEM.u.i extra 0x00 bytes at the end. +** Mem.u.nZero extra 0x00 bytes at the end. ** ** * MEM_Int Integer stored in Mem.u.i. ** @@ -557,7 +557,10 @@ struct PreUpdate { Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ sqlite3_value **apDflt; /* Array of default values, if required */ - u8 keyinfoSpace[SZ_KEYINFO_0]; /* Space to hold pKeyinfo[0] content */ + union { + KeyInfo sKey; + u8 keyinfoSpace[SZ_KEYINFO_0]; /* Space to hold pKeyinfo[0] content */ + } uKey; }; /* @@ -721,9 +724,11 @@ int sqlite3VdbeCheckMemInvariants(Mem*); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY -int sqlite3VdbeCheckFk(Vdbe *, int); +int sqlite3VdbeCheckFkImmediate(Vdbe*); +int sqlite3VdbeCheckFkDeferred(Vdbe*); #else -# define sqlite3VdbeCheckFk(p,i) 0 +# define sqlite3VdbeCheckFkImmediate(p) 0 +# define sqlite3VdbeCheckFkDeferred(p) 0 #endif #ifdef SQLITE_DEBUG diff --git a/src/vdbeapi.c b/src/vdbeapi.c index f5260e7e6..af90d4497 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2192,6 +2192,9 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ } if( p->pPk ){ iStore = sqlite3TableColumnToIndex(p->pPk, iIdx); + }else if( iIdx >= p->pTab->nCol ){ + rc = SQLITE_MISUSE_BKPT; + goto preupdate_old_out; }else{ iStore = sqlite3TableColumnToStorage(p->pTab, iIdx); } @@ -2347,6 +2350,8 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ } if( p->pPk && p->op!=SQLITE_UPDATE ){ iStore = sqlite3TableColumnToIndex(p->pPk, iIdx); + }else if( iIdx >= p->pTab->nCol ){ + return SQLITE_MISUSE_BKPT; }else{ iStore = sqlite3TableColumnToStorage(p->pTab, iIdx); } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8a900aeff..c8b86e6f6 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2994,10 +2994,12 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) || nTrans<=1 ){ - for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + if( needXcommit ){ + for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeTxnState(pBt)>=SQLITE_TXN_WRITE ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + } } } @@ -3008,7 +3010,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ */ for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ){ + int txn = sqlite3BtreeTxnState(pBt); + if( txn!=SQLITE_TXN_NONE ){ + assert( needXcommit || txn==SQLITE_TXN_READ ); rc = sqlite3BtreeCommitPhaseTwo(pBt, 0); } } @@ -3263,28 +3267,31 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ /* -** This function is called when a transaction opened by the database +** These functions are called when a transaction opened by the database ** handle associated with the VM passed as an argument is about to be -** committed. If there are outstanding deferred foreign key constraint -** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. +** committed. If there are outstanding foreign key constraint violations +** return an error code. Otherwise, SQLITE_OK. ** ** If there are outstanding FK violations and this function returns -** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY -** and write an error message to it. Then return SQLITE_ERROR. +** non-zero, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY +** and write an error message to it. */ #ifndef SQLITE_OMIT_FOREIGN_KEY -int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ +static SQLITE_NOINLINE int vdbeFkError(Vdbe *p){ + p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; + p->errorAction = OE_Abort; + sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; + return SQLITE_CONSTRAINT_FOREIGNKEY; +} +int sqlite3VdbeCheckFkImmediate(Vdbe *p){ + if( p->nFkConstraint==0 ) return SQLITE_OK; + return vdbeFkError(p); +} +int sqlite3VdbeCheckFkDeferred(Vdbe *p){ sqlite3 *db = p->db; - if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) - || (!deferred && p->nFkConstraint>0) - ){ - p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; - p->errorAction = OE_Abort; - sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; - return SQLITE_CONSTRAINT_FOREIGNKEY; - } - return SQLITE_OK; + if( (db->nDeferredCons+db->nDeferredImmCons)==0 ) return SQLITE_OK; + return vdbeFkError(p); } #endif @@ -3378,7 +3385,7 @@ int sqlite3VdbeHalt(Vdbe *p){ /* Check for immediate foreign key violations. */ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - (void)sqlite3VdbeCheckFk(p, 0); + (void)sqlite3VdbeCheckFkImmediate(p); } /* If the auto-commit flag is set and this is the only active writer @@ -3392,7 +3399,7 @@ int sqlite3VdbeHalt(Vdbe *p){ && db->nVdbeWrite==(p->readOnly==0) ){ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - rc = sqlite3VdbeCheckFk(p, 1); + rc = sqlite3VdbeCheckFkDeferred(p); if( rc!=SQLITE_OK ){ if( NEVER(p->readOnly) ){ sqlite3VdbeLeave(p); @@ -4257,15 +4264,15 @@ void sqlite3VdbeRecordUnpack( pMem->z = 0; sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); d += sqlite3VdbeSerialTypeLen(serial_type); - pMem++; if( (++u)>=p->nField ) break; + pMem++; } if( d>(u32)nKey && u ){ assert( CORRUPT_DB ); /* In a corrupt record entry, the last pMem might have been set up using ** uninitialized memory. Overwrite its value with NULL, to prevent ** warnings from MSAN. */ - sqlite3VdbeMemSetNull(pMem-1); + sqlite3VdbeMemSetNull(pMem-(u<p->nField)); } testcase( u == pKeyInfo->nKeyField + 1 ); testcase( u < pKeyInfo->nKeyField + 1 ); @@ -4436,6 +4443,32 @@ static void vdbeAssertFieldCountWithinLimits( ** or positive value if *pMem1 is less than, equal to or greater than ** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". */ +static SQLITE_NOINLINE int vdbeCompareMemStringWithEncodingChange( + const Mem *pMem1, + const Mem *pMem2, + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ +){ + int rc; + const void *v1, *v2; + Mem c1; + Mem c2; + sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); + sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + if( (v1==0 || v2==0) ){ + if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; + rc = 0; + }else{ + rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); + } + sqlite3VdbeMemReleaseMalloc(&c1); + sqlite3VdbeMemReleaseMalloc(&c2); + return rc; +} static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, @@ -4447,25 +4480,7 @@ static int vdbeCompareMemString( ** comparison function directly */ return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); }else{ - int rc; - const void *v1, *v2; - Mem c1; - Mem c2; - sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); - sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); - sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); - sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); - v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); - v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); - if( (v1==0 || v2==0) ){ - if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; - rc = 0; - }else{ - rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); - } - sqlite3VdbeMemReleaseMalloc(&c1); - sqlite3VdbeMemReleaseMalloc(&c2); - return rc; + return vdbeCompareMemStringWithEncodingChange(pMem1,pMem2,pColl,prcErr); } } @@ -5378,6 +5393,7 @@ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ } } +#ifndef SQLITE_OMIT_DATETIME_FUNCS /* ** Cause a function to throw an error if it was call from OP_PureFunc ** rather than OP_Function. @@ -5411,6 +5427,7 @@ int sqlite3NotPureFunc(sqlite3_context *pCtx){ } return 1; } +#endif /* SQLITE_OMIT_DATETIME_FUNCS */ #if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) /* @@ -5521,7 +5538,7 @@ void sqlite3VdbePreUpdateHook( preupdate.pCsr = pCsr; preupdate.op = op; preupdate.iNewReg = iReg; - preupdate.pKeyinfo = (KeyInfo*)&preupdate.keyinfoSpace; + preupdate.pKeyinfo = &preupdate.uKey.sKey; preupdate.pKeyinfo->db = db; preupdate.pKeyinfo->enc = ENC(db); preupdate.pKeyinfo->nKeyField = pTab->nCol; diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 42edcf7de..a15fec6c4 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -385,7 +385,7 @@ static int blobReadWrite( int iOffset, int (*xCall)(BtCursor*, u32, u32, void*) ){ - int rc; + int rc = SQLITE_OK; Incrblob *p = (Incrblob *)pBlob; Vdbe *v; sqlite3 *db; @@ -425,17 +425,32 @@ static int blobReadWrite( ** using the incremental-blob API, this works. For the sessions module ** anyhow. */ - sqlite3_int64 iKey; - iKey = sqlite3BtreeIntegerKey(p->pCsr); - assert( v->apCsr[0]!=0 ); - assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); - sqlite3VdbePreUpdateHook( - v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol - ); + if( sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ){ + /* If the cursor is not currently valid, try to reseek it. This + ** always either fails or finds the correct row - the cursor will + ** have been marked permanently CURSOR_INVALID if the open row has + ** been deleted. */ + int bDiff = 0; + rc = sqlite3BtreeCursorRestore(p->pCsr, &bDiff); + assert( bDiff==0 || sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ); + } + if( sqlite3BtreeCursorIsValidNN(p->pCsr) ){ + sqlite3_int64 iKey; + iKey = sqlite3BtreeIntegerKey(p->pCsr); + assert( v->apCsr[0]!=0 ); + assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); + sqlite3VdbePreUpdateHook( + v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol + ); + } } + if( rc==SQLITE_OK ){ + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); + } +#else + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); #endif - rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); sqlite3BtreeLeaveCursor(p->pCsr); if( rc==SQLITE_ABORT ){ sqlite3VdbeFinalize(v); diff --git a/src/where.c b/src/where.c index 11e24a8d3..b60c4d1c0 100644 --- a/src/where.c +++ b/src/where.c @@ -1201,7 +1201,9 @@ static SQLITE_NOINLINE void constructAutomaticIndex( VdbeCoverage(v); VdbeComment((v, "next row of %s", pSrc->pSTab->zName)); }else{ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + assert( pLevel->addrHalt ); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind,pLevel->iTabCur,pLevel->addrHalt); + VdbeCoverage(v); } if( pPartial ){ iContinue = sqlite3VdbeMakeLabel(pParse); @@ -1229,11 +1231,14 @@ static SQLITE_NOINLINE void constructAutomaticIndex( pSrc->u4.pSubq->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pSrc->fg.viaCoroutine = 0; + sqlite3VdbeJumpHere(v, addrTop); }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + if( (pSrc->fg.jointype & JT_LEFT)!=0 ){ + sqlite3VdbeJumpHere(v, addrTop); + } } - sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); /* Jump here when skipping the initialization */ @@ -3234,6 +3239,7 @@ static int whereLoopAddBtreeIndex( if( ExprUseXSelect(pExpr) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ int i; + int bRedundant = 0; nIn = 46; assert( 46==sqlite3LogEst(25) ); /* The expression may actually be of the form (x, y) IN (SELECT...). @@ -3242,7 +3248,20 @@ static int whereLoopAddBtreeIndex( ** for each such term. The following loop checks that pTerm is the ** first such term in use, and sets nIn back to 0 if it is not. */ for(i=0; i<pNew->nLTerm-1; i++){ - if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; + if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ){ + nIn = 0; + if( pNew->aLTerm[i]->u.x.iField == pTerm->u.x.iField ){ + /* Detect when two or more columns of an index match the same + ** column of a vector IN operater, and avoid adding the column + ** to the WhereLoop more than once. See tag-20250707-01 + ** in test/rowvalue.test */ + bRedundant = 1; + } + } + } + if( bRedundant ){ + pNew->nLTerm--; + continue; } }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ @@ -3474,7 +3493,7 @@ static int whereLoopAddBtreeIndex( if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<pProbe->nColumn && (pNew->u.btree.nEq<pProbe->nKeyCol || - (pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY && !pProbe->bIdxRowid)) + pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ if( pNew->u.btree.nEq>3 ){ sqlite3ProgressCheck(pParse); @@ -3513,6 +3532,7 @@ static int whereLoopAddBtreeIndex( && pProbe->hasStat1!=0 && OptimizationEnabled(db, SQLITE_SkipScan) && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && pSrc->fg.fromExists==0 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -4017,6 +4037,7 @@ static int whereLoopAddBtree( pNew->u.btree.nEq = 0; pNew->u.btree.nBtm = 0; pNew->u.btree.nTop = 0; + pNew->u.btree.nDistinctCol = 0; pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; @@ -5085,8 +5106,6 @@ static i8 wherePathSatisfiesOrderBy( obSat = obDone; } break; - }else if( wctrlFlags & WHERE_DISTINCTBY ){ - pLoop->u.btree.nDistinctCol = 0; } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; @@ -7070,6 +7089,14 @@ WhereInfo *sqlite3WhereBegin( pTab = pTabItem->pSTab; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; + pLevel->addrBrk = sqlite3VdbeMakeLabel(pParse); + if( ii==0 || (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ + pLevel->addrHalt = pLevel->addrBrk; + }else if( pWInfo->a[ii-1].pRJ ){ + pLevel->addrHalt = pWInfo->a[ii-1].addrBrk; + }else{ + pLevel->addrHalt = pWInfo->a[ii-1].addrHalt; + } if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ /* Do nothing */ }else @@ -7121,6 +7148,13 @@ WhereInfo *sqlite3WhereBegin( sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, (const u8*)&pTabItem->colUsed, P4_INT64); #endif + if( ii>=2 + && (pTabItem[0].fg.jointype & (JT_LTORJ|JT_LEFT))==0 + && pLevel->addrHalt==pWInfo->a[0].addrHalt + ){ + sqlite3VdbeAddOp2(v, OP_IfEmpty, pTabItem->iCursor, pWInfo->iBreak); + VdbeCoverage(v); + } }else{ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } @@ -7377,6 +7411,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ + if( pTabList->a[pLevel->iFrom].fg.fromExists ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + } /* The common case: Advance to the next row */ if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); diff --git a/src/whereInt.h b/src/whereInt.h index 3a9353e07..09e02c8c7 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -75,6 +75,7 @@ struct WhereLevel { int iTabCur; /* The VDBE cursor used to access the table */ int iIdxCur; /* The VDBE cursor used to access pIdx */ int addrBrk; /* Jump here to break out of the loop */ + int addrHalt; /* Abort the query due to empty table or similar */ int addrNxt; /* Jump here to start the next IN combination */ int addrSkip; /* Jump here for next iteration of skip-scan */ int addrCont; /* Jump here to continue with the next loop cycle */ diff --git a/src/wherecode.c b/src/wherecode.c index 9581ac389..1efa34a5d 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -126,7 +126,6 @@ void sqlite3WhereAddExplainText( #endif { VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe, addr); - SrcItem *pItem = &pTabList->a[pLevel->iFrom]; sqlite3 *db = pParse->db; /* Database handle */ int isSearch; /* True for a SEARCH. False for SCAN. */ @@ -149,7 +148,10 @@ void sqlite3WhereAddExplainText( sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); str.printfFlags = SQLITE_PRINTF_INTERNAL; - sqlite3_str_appendf(&str, "%s %S", isSearch ? "SEARCH" : "SCAN", pItem); + sqlite3_str_appendf(&str, "%s %S%s", + isSearch ? "SEARCH" : "SCAN", + pItem, + pItem->fg.fromExists ? " EXISTS" : ""); if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ const char *zFmt = 0; Index *pIdx; @@ -598,7 +600,9 @@ static Expr *removeUnindexableInClauseTerms( int iField; assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); iField = pLoop->aLTerm[i]->u.x.iField - 1; - if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ + if( NEVER(pOrigRhs->a[iField].pExpr==0) ){ + continue; /* Duplicate PK column */ + } pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); pOrigRhs->a[iField].pExpr = 0; if( pRhs ) pRhs->a[pRhs->nExpr-1].u.x.iOrderByCol = iField+1; @@ -695,7 +699,7 @@ static SQLITE_NOINLINE void codeINTerm( return; } } - for(i=iEq;i<pLoop->nLTerm; i++){ + for(i=iEq; i<pLoop->nLTerm; i++){ assert( pLoop->aLTerm[i]!=0 ); if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; } @@ -704,22 +708,13 @@ static SQLITE_NOINLINE void codeINTerm( if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){ eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); }else{ - Expr *pExpr = pTerm->pExpr; - if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){ - sqlite3 *db = pParse->db; - pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); - if( !db->mallocFailed ){ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab); - pExpr->iTable = iTab; - } - sqlite3ExprDelete(db, pX); - }else{ - int n = sqlite3ExprVectorSize(pX->pLeft); - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); + sqlite3 *db = pParse->db; + Expr *pXMod = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); + if( !db->mallocFailed ){ + aiMap = (int*)sqlite3DbMallocZero(db, sizeof(int)*nEq); + eType = sqlite3FindInIndex(pParse, pXMod, IN_INDEX_LOOP, 0, aiMap, &iTab); } - pX = pExpr; + sqlite3ExprDelete(db, pXMod); } if( eType==IN_INDEX_INDEX_DESC ){ @@ -749,7 +744,7 @@ static SQLITE_NOINLINE void codeINTerm( if( pIn ){ int iMap = 0; /* Index in aiMap[] */ pIn += i; - for(i=iEq;i<pLoop->nLTerm; i++){ + for(i=iEq; i<pLoop->nLTerm; i++){ if( pLoop->aLTerm[i]->pExpr==pX ){ int iOut = iTarget + i - iEq; if( eType==IN_INDEX_ROWID ){ @@ -1400,6 +1395,7 @@ static SQLITE_NOINLINE void filterPullDown( int addrNxt, /* Jump here to bypass inner loops */ Bitmask notReady /* Loops that are not ready */ ){ + int saved_addrBrk; while( ++iLevel < pWInfo->nLevel ){ WhereLevel *pLevel = &pWInfo->a[iLevel]; WhereLoop *pLoop = pLevel->pWLoop; @@ -1408,7 +1404,7 @@ static SQLITE_NOINLINE void filterPullDown( /* ,--- Because sqlite3ConstructBloomFilter() has will not have set ** vvvvv--' pLevel->regFilter if this were true. */ if( NEVER(pLoop->prereq & notReady) ) continue; - assert( pLevel->addrBrk==0 ); + saved_addrBrk = pLevel->addrBrk; pLevel->addrBrk = addrNxt; if( pLoop->wsFlags & WHERE_IPK ){ WhereTerm *pTerm = pLoop->aLTerm[0]; @@ -1438,7 +1434,7 @@ static SQLITE_NOINLINE void filterPullDown( VdbeCoverage(pParse->pVdbe); } pLevel->regFilter = 0; - pLevel->addrBrk = 0; + pLevel->addrBrk = saved_addrBrk; } } @@ -1485,7 +1481,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( sqlite3 *db; /* Database connection */ SrcItem *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ - int addrHalt; /* addrBrk for the outermost loop */ int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ @@ -1529,7 +1524,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** there are no IN operators in the constraints, the "addrNxt" label ** is the same as "addrBrk". */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + addrBrk = pLevel->addrNxt = pLevel->addrBrk; addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); /* If this is the right table of a LEFT OUTER JOIN, allocate and @@ -1545,14 +1540,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( VdbeComment((v, "init LEFT JOIN match flag")); } - /* Compute a safe address to jump to if we discover that the table for - ** this loop is empty and can never contribute content. */ - for(j=iLevel; j>0; j--){ - if( pWInfo->a[j].iLeftJoin ) break; - if( pWInfo->a[j].pRJ ) break; - } - addrHalt = pWInfo->a[j].addrBrk; - /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ int regYield; @@ -1791,7 +1778,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, pX->op==TK_GE); sqlite3ReleaseTempReg(pParse, rTemp); }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, pLevel->addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); } @@ -2586,7 +2573,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( codeCursorHint(pTabItem, pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt); + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev],iCur,pLevel->addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; @@ -2858,7 +2845,10 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( WhereLoop *pLoop = pLevel->pWLoop; SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; SrcList *pFrom; - u8 fromSpace[SZ_SRCLIST_1]; + union { + SrcList sSrc; + u8 fromSpace[SZ_SRCLIST_1]; + } uSrc; Bitmask mAll = 0; int k; @@ -2902,7 +2892,7 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); } } - pFrom = (SrcList*)fromSpace; + pFrom = &uSrc.sSrc; pFrom->nSrc = 1; pFrom->nAlloc = 1; memcpy(&pFrom->a[0], pTabItem, sizeof(SrcItem)); diff --git a/src/whereexpr.c b/src/whereexpr.c index 53c8508e5..0d17b0d75 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -948,7 +948,7 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr, SrcList *pSrc){ if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* (3) */ assert( pSrc!=0 ); if( pExpr->op==TK_IS - && pSrc->nSrc + && pSrc->nSrc>=2 && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ return 0; /* (4) */ @@ -1541,7 +1541,7 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; + pNewTerm->prereqRight = prereqExpr | extraRight; pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.x.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_AUX; @@ -1652,7 +1652,7 @@ static void whereAddLimitExpr( ** ** 1. The SELECT statement has a LIMIT clause, and ** 2. The SELECT statement is not an aggregate or DISTINCT query, and -** 3. The SELECT statement has exactly one object in its from clause, and +** 3. The SELECT statement has exactly one object in its FROM clause, and ** that object is a virtual table, and ** 4. There are no terms in the WHERE clause that will not be passed ** to the virtual table xBestIndex method. @@ -1689,8 +1689,22 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ ** (leftCursor==iCsr) test below. */ continue; } - if( pWC->a[ii].leftCursor!=iCsr ) return; - if( pWC->a[ii].prereqRight!=0 ) return; + if( pWC->a[ii].leftCursor==iCsr && pWC->a[ii].prereqRight==0 ) continue; + + /* If this term has a parent with exactly one child, and the parent will + ** be passed through to xBestIndex, then this term can be ignored. */ + if( pWC->a[ii].iParent>=0 ){ + WhereTerm *pParent = &pWC->a[ pWC->a[ii].iParent ]; + if( pParent->leftCursor==iCsr + && pParent->prereqRight==0 + && pParent->nChild==1 + ){ + continue; + } + } + + /* This term will not be passed through. Do not add a LIMIT clause. */ + return; } /* Check condition (5). Return early if it is not met. */ diff --git a/test/altertab2.test b/test/altertab2.test index 56e42f1a6..f2a1d74c4 100644 --- a/test/altertab2.test +++ b/test/altertab2.test @@ -358,7 +358,7 @@ do_catchsql_test 8.6 { CREATE INDEX i0 ON t0(likelihood(1,2) AND 0); ALTER TABLE t0 RENAME TO t1; SELECT sql FROM sqlite_master WHERE name='i0'; -} {1 {error in index i0: second argument to likelihood() must be a constant between 0.0 and 1.0}} +} {1 {second argument to likelihood() must be a constant between 0.0 and 1.0}} reset_db diff --git a/test/altertab3.test b/test/altertab3.test index 5f5c11b0b..92060fb41 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -190,14 +190,14 @@ do_execsql_test 8.1 { } do_execsql_test 8.2.1 { CREATE TABLE t2 (c0); - CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 100) IN ())); + CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 1.0) IN ())); ALTER TABLE t2 RENAME COLUMN c0 TO c1; } do_execsql_test 8.2.2 { SELECT sql FROM sqlite_master WHERE tbl_name = 't2'; } { {CREATE TABLE t2 (c1)} - {CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 100) IN ()))} + {CREATE INDEX i2 ON t2((LIKELIHOOD(c1, 1.0) IN ()))} } do_test 8.2.3 { sqlite3 db2 test.db @@ -662,14 +662,6 @@ do_execsql_test 27.2 { {CREATE TABLE t1(a, b AS ((WITH w1 (xyz) AS ( SELECT t1.b FROM t1 ) SELECT 123) IN ()))} } -do_execsql_test 27.3 { - CREATE TABLE t0(c0 , c1 AS (CASE TRUE NOT IN () WHEN NULL THEN CASE + 0xa ISNULL WHEN NOT + 0x9 THEN t0.c1 ELSE CURRENT_TIME LIKE CAST (t0.c1 REGEXP '-([1-9]\d*.\d*|0\.\d*[1-9]\d*)'ESCAPE (c1) COLLATE BINARY BETWEEN c1 AND c1 NOT IN (WITH t4 (c0) AS (WITH t3 (c0) AS NOT MATERIALIZED (WITH RECURSIVE t2 (c0) AS (WITH RECURSIVE t1 AS (VALUES (x'717171ff71717171' ) ) SELECT DISTINCT t0.c0 FROM t0 NOT INDEXED WHERE t0.c0 =t0.c0 GROUP BY 0x9 ) SELECT DISTINCT t0.c0 FROM t0 NOT INDEXED WHERE t0.c0 =t0.c1 ) SELECT DISTINCT t0.c0 FROM t0 NOT INDEXED WHERE t0.c0 =t0.c0 GROUP BY typeof(0x9 ) ) SELECT DISTINCT t0.c0 FROM t0 NOT INDEXED WHERE t0.c0 =t0.c0 GROUP BY typeof(typeof(0x9 ) ) ) IN t0 BETWEEN typeof(typeof(typeof(hex(*) FILTER (WHERE + x'5ccd1e68' ) ) ) ) AND 1 >0xa AS BLOB (+4.4E4 , -0xe ) ) END <> c1 IN () END ) VIRTUAL , c35 PRIMARY KEY , c60 , c64 NUMERIC (-6.8 , -0xE ) ) WITHOUT ROWID ; -} {} - -do_execsql_test 27.4 { - ALTER TABLE t0 DROP COLUMN c60; -} {} - #------------------------------------------------------------------------- reset_db do_execsql_test 28.1 { diff --git a/test/bestindexC.test b/test/bestindexC.test index 48f3a2765..8b96a19e6 100644 --- a/test/bestindexC.test +++ b/test/bestindexC.test @@ -349,4 +349,78 @@ do_execsql_test 5.9 { three six seven } +#-------------------------------------------------------------------------- + +reset_db +register_tcl_module db + +proc quote {str} { + return "'[string map {' ''} $str]'" +} + +proc vtab_command {lVal method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c, d)" + } + + xBestIndex { + set hdl [lindex $args 0] + set clist [$hdl constraints] + + set idx 0 + set idxnum 0 + + foreach c $clist { + array set a $c + if {$a(usable)==0} continue + + if {$a(op)=="limit"} { + set idxnum [$hdl rhs_value $idx 555] + } + + incr idx + } + + return "cost 1000 rows 1000 idxnum $idxnum" + + } + + xFilter { + foreach {idxnum idxstr lArg} $args {} + return [list sql "SELECT 0, $idxnum, $idxnum, $idxnum, $idxnum"] + } + } + + return {} +} + +do_execsql_test 6.0 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(2, 2); + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command t1); +} + +do_execsql_test 6.1 { SELECT * FROM x1 LIMIT 50 } {50 50 50 50} + +do_execsql_test 6.2 { SELECT * FROM x1 WHERE b=c LIMIT 5 } {0 0 0 0} + +do_execsql_test 6.3 { + SELECT (SELECT a FROM x1 WHERE t1.x=t1.y LIMIT 10) FROM t1 +} {0} + +do_execsql_test 6.4 { + SELECT (SELECT a FROM x1 WHERE x1.a=1) FROM t1 +} {1} + +do_execsql_test 6.5 { + SELECT (SELECT a FROM x1 WHERE x1.a=1 LIMIT 1) FROM t1 +} {1} + +do_execsql_test 6.6 { + SELECT (SELECT a FROM x1 WHERE x1.a=555 LIMIT 2) FROM t1 +} {555} + finish_test + + diff --git a/test/between.test b/test/between.test index 16c3913d1..5e02ef9b2 100644 --- a/test/between.test +++ b/test/between.test @@ -140,4 +140,21 @@ foreach {tn expr res} { do_execsql_test between-2.1.$tn $sql $res } +#------------------------------------------------------------------------- +reset_db +do_execsql_test between-3.0 { + CREATE TABLE t1(x, y); + CREATE INDEX i1 ON t1(x); + INSERT INTO t1 VALUES(4, 4); + CREATE TABLE t2(a, b); +} + +do_execsql_test between-3.1 { + SELECT * FROM t1 LEFT JOIN t2 ON (x BETWEEN 1 AND 3); +} {4 4 {} {}} + +do_execsql_test between-3.2 { + SELECT * FROM t1 LEFT JOIN t2 ON (x BETWEEN 5 AND 7); +} {4 4 {} {}} + finish_test diff --git a/test/eqp.test b/test/eqp.test index 5d2659be7..147b5ceaf 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -338,8 +338,7 @@ det 3.3.3 { } { QUERY PLAN |--SCAN t1 - `--CORRELATED SCALAR SUBQUERY xxxxxx - `--SCAN t2 + `--SCAN t2 EXISTS } #------------------------------------------------------------------------- diff --git a/test/existsexpr.test b/test/existsexpr.test new file mode 100644 index 000000000..d02f8c5c1 --- /dev/null +++ b/test/existsexpr.test @@ -0,0 +1,432 @@ +# 2024 May 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix existsexpr + + +do_execsql_test 1.0 { + CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID; + INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6); + CREATE INDEX x1b ON x1(b); + + CREATE TABLE x2(x, y); + INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6); +} + +do_execsql_test 1.1 { + SELECT 1 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=5) +} {1} + +do_execsql_test 1.2 { + SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) +} {1 2 3 4 5 6} + +# With "a=x", the UNIQUE index means the EXIST can be transformed to a join. +# So no "SUBQUERY". With "b=x", the index is not UNIQUE and so there is a +# "SUBQUERY". +do_execsql_test 1.3.1 { + EXPLAIN QUERY PLAN + SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) +} {~/SUBQUERY/} +do_execsql_test 1.3.2 { + EXPLAIN QUERY PLAN + SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE b=x) +} {~/SUBQUERY/} + +do_execsql_test 1.4.1 { + EXPLAIN QUERY PLAN + SELECT * FROM x2 WHERE x=1 AND EXISTS (SELECT 1 FROM x1 WHERE a=x) +} {~/SUBQUERY/} +do_execsql_test 1.4.2 { + EXPLAIN QUERY PLAN + SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) AND y=2 +} {~/SUBQUERY/} + +do_execsql_test 1.5 { + SELECT count(*) FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) +} {3} + +#------------------------------------------------------------------------- +do_execsql_test 2.0 { + CREATE TABLE t1(a, b); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) INSERT INTO t1 SELECT i, i FROM s; + + CREATE TABLE t2(c, d); + WITH s(i) AS ( + SELECT 10 UNION ALL SELECT i+10 FROM s WHERE i<1000 + ) INSERT INTO t2 SELECT i, i FROM s; +} + +do_execsql_test 2.1 { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; +} {1000 100} + +do_execsql_test 2.2 { + SELECT count(*) FROM t1, t2 WHERE a=c; +} {100} + +do_execsql_test 2.3 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a) +} {100} +do_eqp_test 2.4 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a) +} {SCAN t1} + +do_execsql_test 2.4.0 { + CREATE UNIQUE INDEX t2c ON t2(c); + CREATE UNIQUE INDEX t1a ON t1(a); +} + +do_eqp_test 2.4.1 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a); +} {SCAN t1*t2 EXISTS} +do_execsql_test 2.4.2 { + ANALYZE; +} +do_eqp_test 2.4.3 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a); +} {SCAN t1*t2 EXISTS} +do_execsql_test 2.4.4 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a); +} {100} + +do_execsql_test 2.5.1 { + EXPLAIN QUERY PLAN + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.rowid=a); +} {~/SUBQUERY/} + +#------------------------------------------------------------------------- +proc do_subquery_test {tn bSub sql res} { + set r1(0) ~/SUBQUERY/ + set r1(1) /SUBQUERY/ + do_execsql_test $tn.1 "explain query plan $sql" $r1($bSub) + do_execsql_test $tn.2 $sql $res +} + +do_execsql_test 3.0 { + CREATE TABLE y1(a, b, c); + CREATE TABLE y2(x, y, z); + CREATE UNIQUE INDEX y2zy ON y2(z, y); + + INSERT INTO y1 VALUES(1, 1, 1); + INSERT INTO y1 VALUES(2, 2, 2); + INSERT INTO y1 VALUES(3, 3, 3); + INSERT INTO y1 VALUES(4, 4, 4); + + INSERT INTO y2 VALUES(1, 1, 1); + INSERT INTO y2 VALUES(3, 3, 3); +} + +do_subquery_test 3.1 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=a AND y=b AND x=z + ) +} { + 1 1 1 3 3 3 +} + +do_subquery_test 3.2 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=max(a,b) AND y=min(b,a) AND x=z + ) +} { + 1 1 1 3 3 3 +} + +do_subquery_test 3.3 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=max(a,b) AND y=min(b,a) AND c!=3 + ) +} { + 1 1 1 +} + +do_subquery_test 3.4 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=max(a,b) AND b=3 + ) +} { + 3 3 3 +} + +do_subquery_test 3.5 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=a-1 AND y=a-1 + ) +} { + 2 2 2 + 4 4 4 +} + +do_subquery_test 3.6 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=a-1 AND y+1=a + ) +} { + 2 2 2 + 4 4 4 +} + +do_subquery_test 3.7 1 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT count(*) FROM y2 WHERE z=a-1 AND y=a-1 + ) +} { + 1 1 1 + 2 2 2 + 3 3 3 + 4 4 4 +} + +do_subquery_test 3.8 0 { + SELECT * FROM y1 WHERE EXISTS ( SELECT a+1 FROM y2 ) +} { + 1 1 1 + 2 2 2 + 3 3 3 + 4 4 4 +} + +do_subquery_test 3.9 1 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 one, y2 two WHERE one.z=a-1 AND one.y=a-1 + ) +} { + 2 2 2 + 4 4 4 +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE tx1(a TEXT COLLATE nocase, b TEXT); + CREATE UNIQUE INDEX tx1ab ON tx1(a, b); + + INSERT INTO tx1 VALUES('a', 'a'); + INSERT INTO tx1 VALUES('B', 'b'); + INSERT INTO tx1 VALUES('c', 'c'); + INSERT INTO tx1 VALUES('D', 'd'); + INSERT INTO tx1 VALUES('e', 'e'); + + CREATE TABLE tx2(x, y); + INSERT INTO tx2 VALUES('A', 'a'); + INSERT INTO tx2 VALUES('b', 'b'); + INSERT INTO tx2 VALUES('C', 'c'); + INSERT INTO tx2 VALUES('D', 'd'); +} + +do_subquery_test 4.1 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND b=y + ) +} { + A a + b b + C c + D d +} + +do_subquery_test 4.1.1 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE (a COLLATE nocase)=x AND b=y + ) +} { + A a b b C c D d +} +do_subquery_test 4.1.2 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND (b COLLATE binary)=y + ) +} { + A a b b C c D d +} +do_subquery_test 4.1.1 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE x=(a COLLATE nocase) AND b=y + ) +} { + A a b b C c D d +} +do_subquery_test 4.1.2 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND y=(b COLLATE binary) + ) +} { + A a b b C c D d +} + +do_subquery_test 4.2 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND b=y COLLATE nocase + ) +} { + A a + b b + C c + D d +} + +do_execsql_test 4.3 { + DROP INDEX tx1ab; + CREATE UNIQUE INDEX tx1ab ON tx1(a COLLATE binary, b); +} + +do_subquery_test 4.4 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND b=y + ) +} { + A a + b b + C c + D d +} + +do_subquery_test 4.4 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x COLLATE binary AND b=y + ) +} { + D d +} + +do_subquery_test 4.4 1 { + SELECT EXISTS ( SELECT x FROM tx1 ) FROM tx2 +} { + 1 1 1 1 +} + +do_subquery_test 4.4 1 { + SELECT (SELECT EXISTS ( SELECT x FROM tx1 ) WHERE 1) FROM tx2 +} { + 1 1 1 1 +} + +#------------------------------------------------------------------------- +proc cols {s f} { + set lCols [list] + for {set i $s} {$i<=$f} {incr i} { + lappend lCols [format "c%02d" $i] + } + join $lCols ", " +} +proc vals {n val} { + set lVal [list] + for {set i 0} {$i<$n} {incr i} { + lappend lVal $val + } + join $lVal ", " +} +proc exprs {s f} { + set lExpr [list] + for {set i $s} {$i<=$f} {incr i} { + lappend lExpr [format "c%02d = o" $i] + } + join $lExpr " AND " +} + + +do_execsql_test 5.0 " + CREATE TABLE a1( [cols 0 99] ); +" +do_execsql_test 5.1 " + -- 63 column index + CREATE UNIQUE INDEX a1idx1 ON a1( [cols 0 62] ); +" +do_execsql_test 5.2 " + -- 64 column index + CREATE UNIQUE INDEX a1idx2 ON a1( [cols 10 73] ); +" +do_execsql_test 5.2 " + -- 65 column index + CREATE UNIQUE INDEX a1idx3 ON a1( [cols 20 84] ); +" + +do_test 5.3 { + foreach v {1 2 3 4 5 6} { + execsql "INSERT INTO a1 VALUES( [vals 100 $v] )" + } +} {} + +do_execsql_test 5.4 { + CREATE TABLE a2(o); + INSERT INTO a2 VALUES(2), (5); +} + +do_subquery_test 5.5 0 " + SELECT o FROM a2 WHERE EXISTS ( + SELECT 1 FROM a1 WHERE [exprs 0 62] + ) +" { + 2 5 +} + +do_subquery_test 5.6 0 " + SELECT o FROM a2 WHERE EXISTS ( + SELECT 1 FROM a1 WHERE [exprs 10 73] + ) +" { + 2 5 +} + +do_subquery_test 5.7 0 " + SELECT o FROM a2 WHERE EXISTS ( + SELECT 1 FROM a1 WHERE [exprs 20 84] + ) +" { + 2 5 +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(a, b UNIQUE, c UNIQUE); + CREATE TABLE t2(a INfEGER PRIMARY KEY, b); + CREATE UNIQUE INDEX t2b ON t2(b); +} + +do_catchsql_test 6.1 { + SELECT a FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c COLLATE f = a) +} {1 {no such collation sequence: f}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE TABLE t1(x); + CREATE TABLE t2(y UNIQUE); + + INSERT INTO t1 VALUES(1), (2); + INSERT INTO t2 VALUES(1), (3); + + SELECT * FROM t1 one LEFT JOIN t1 two ON (one.x=two.x AND EXISTS ( + SELECT 1 FROM t2 WHERE y=one.x + )); +} { + 1 1 + 2 {} +} + +# https://sqlite.org/forum/forumpost/2025-07-23T10:59:14z +reset_db +do_execsql_test 8.0 { + CREATE TABLE t0 (c0 INT); INSERT INTO t0(c0) VALUES (1); + CREATE TABLE t1(c0 INT); INSERT INTO t1(c0) VALUES (2); + SELECT * FROM t1 WHERE EXISTS (SELECT 1 FROM t0 LIMIT 0); +} {} + +finish_test diff --git a/test/existsexpr2.test b/test/existsexpr2.test new file mode 100644 index 000000000..f7644bf80 --- /dev/null +++ b/test/existsexpr2.test @@ -0,0 +1,96 @@ +# 2024 June 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix existsexpr2 + + +do_execsql_test 1.0 { + CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID; + INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6); + CREATE INDEX x1b ON x1(b); + + CREATE TABLE x2(x, y); + INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6); +} + +do_execsql_test 1.1 { + SELECT * FROM x1 WHERE EXISTS (SELECT 1 FROM x2 WHERE a!=123) +} {1 2 3 4 5 6} + +do_execsql_test 1.2 { + CREATE TABLE x3(u, v); + CREATE INDEX x3u ON x3(u); + INSERT INTO x3 VALUES + (1, 1), (1, 2), (1, 3), + (2, 1), (2, 2), (2, 3); +} + +do_execsql_test 1.3 { + SELECT * FROM x1 WHERE EXISTS ( + SELECT 1 FROM x3 WHERE u IN (1, 2, 3, 4) AND v=b + ); +} { + 1 2 +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1ab ON t1(a,b); + + INSERT INTO t1 VALUES + ('abc', 1, 1), + ('abc', 2, 2), + ('abc', 2, 3), + + ('def', 1, 1), + ('def', 2, 2), + ('def', 2, 3); + + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES(1, 1), (2, 2), (3, 3); + + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1 VALUES('t1','t1ab','10000 5000 2'); + ANALYZE sqlite_master; +} + + +do_execsql_test 2.1 { + SELECT a,b,c FROM t1 WHERE b=2 ORDER BY a +} { + abc 2 2 + abc 2 3 + def 2 2 + def 2 3 +} + +do_execsql_test 2.2 { + SELECT x, y FROM t2 WHERE EXISTS ( + SELECT 1 FROM t1 WHERE b=x + ) +} { + 1 1 + 2 2 +} + + + +finish_test + + diff --git a/test/existsfault.test b/test/existsfault.test new file mode 100644 index 000000000..4b335d84c --- /dev/null +++ b/test/existsfault.test @@ -0,0 +1,49 @@ +# 2024 May 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +set testprefix existsfault + +db close +sqlite3_shutdown +sqlite3_config_lookaside 0 0 +sqlite3_initialize +autoinstall_test_functions +sqlite3 db test.db + +do_execsql_test 1.0 { + CREATE TABLE x1(a, b); + INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6); + CREATE UNIQUE INDEX x1a ON x1(a); + CREATE INDEX x1b ON x1(b); + + CREATE TABLE x2(x, y); + INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6); +} + +do_faultsim_test 1 -faults oom* -prep { + sqlite3 db test.db + execsql { SELECT * FROM sqlite_schema } +} -body { + execsql { + SELECT count(*) FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) AND y!=11 + } +} -test { + faultsim_test_result {0 3} +} + +finish_test + + diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 09898d7b3..3868b5538 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -541,9 +541,12 @@ static void blobListLoadFromDb( int n = 0; int rc; char *z2; - unsigned char tmp[SZ_BLOB(8)]; + union { + Blob sBlob; + unsigned char tmp[SZ_BLOB(8)]; + } uBlob; - head = (Blob*)tmp; + head = &uBlob.sBlob; if( firstId>0 ){ z2 = sqlite3_mprintf("%s WHERE rowid BETWEEN %d AND %d", zSql, firstId, lastId); diff --git a/test/hook.test b/test/hook.test index 8638d3a6b..a4256732e 100644 --- a/test/hook.test +++ b/test/hook.test @@ -488,11 +488,21 @@ proc preupdate_hook {args} { set type [lindex $args 0] eval lappend ::preupdate $args if {$type != "INSERT"} { + set x [catch {db preupdate old [db preupdate count]}] + if {!$x} { + lappend "ERROR: sqlite3_preupdate_old() accepted an out-of-bounds\ + column index" + } for {set i 0} {$i < [db preupdate count]} {incr i} { lappend ::preupdate [db preupdate old $i] } } if {$type != "DELETE"} { + set x [catch {db preupdate new [db preupdate count]}] + if {!$x} { + lappend "ERROR: sqlite3_preupdate_old() accepted an out-of-bounds\ + column index" + } for {set i 0} {$i < [db preupdate count]} {incr i} { set rc [catch { db preupdate new $i } v] lappend ::preupdate $v diff --git a/test/incrblob4.test b/test/incrblob4.test index dbff8eb7d..c9bcee8a3 100644 --- a/test/incrblob4.test +++ b/test/incrblob4.test @@ -106,4 +106,102 @@ do_test 4.4 { } {SQLITE_LOCKED} close $blob +#------------------------------------------------------------------------- + +ifcapable preupdate { + +reset_db +do_execsql_test 5.1 { + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + INSERT INTO t2 VALUES(1000, 'abcdefghijklmnopqrstuvwxyz'); + INSERT INTO t2 VALUES(2000, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'); + INSERT INTO t2 VALUES(3000, 'abcdefghijklmnopqrstuvwxyz'); +} + +do_test 5.2.1 { + execsql BEGIN + set blob [db incrblob t2 b 2000] + seek $blob 0 + puts -nonewline $blob "hello " + flush $blob + execsql ROLLBACK +} {} + +do_test 5.2.2 { + puts -nonewline $blob "world" + set rc [catch { flush $blob } msg] + list $rc [regsub {input/output} $msg {I/O}] +} "1 {error flushing \"$blob\": I/O error}" +catch { close $blob } + +set preupdate_count 0 +proc preupdate {args} { incr ::preupdate_count ; return {} } +db preupdate hook preupdate + +set preupdate_count 0 +do_test 5.3.1 { + execsql BEGIN + set blob [db incrblob t2 b 1000] + seek $blob 0 + puts -nonewline $blob "hello " + flush $blob + execsql ROLLBACK +} {} + +do_test 5.3.2 { + puts -nonewline $blob "world" + set rc [catch { flush $blob } msg] + list $rc [regsub {input/output} $msg {I/O}] +} "1 {error flushing \"$blob\": I/O error}" +catch { close $blob } + +do_test 5.3.3 { + set ::preupdate_count +} {1} + +set preupdate_count 0 +do_test 5.4.1 { + execsql BEGIN + set blob [db incrblob t2 b 1000] + seek $blob 0 + puts -nonewline $blob "hello " + flush $blob + execsql { DELETE FROM t2 WHERE a=3000; } +} {} + +do_test 5.4.2 { + puts -nonewline $blob "world" + list [catch { flush $blob } msg] $msg +} "0 {}" +catch { close $blob } +catchsql { ROLLBACK } + +do_test 5.3.3 { + set ::preupdate_count +} {3} + +set preupdate_count 0 +do_test 5.4.3 { + execsql BEGIN + set blob [db incrblob t2 b 2000] + seek $blob 0 + puts -nonewline $blob "hello " + flush $blob + execsql { UPDATE t2 SET b='abcdefghijklmnopqrstuvwxyz' WHERE a=2000 } +} {} + +do_test 5.4.4 { + puts -nonewline $blob "world" + set rc [catch { flush $blob } msg] + list $rc [regsub {input/output} $msg {I/O}] +} "1 {error flushing \"$blob\": I/O error}" +catch { close $blob } +catchsql { ROLLBACK } + +do_test 5.4.5 { + set ::preupdate_count +} {2} + +} + finish_test diff --git a/test/json101.test b/test/json101.test index e22902f86..7582d14a6 100644 --- a/test/json101.test +++ b/test/json101.test @@ -892,15 +892,15 @@ do_execsql_test json101-13.100 { INSERT INTO t2(id,json) VALUES(4,'{"value":4}'); INSERT INTO t2(id,json) VALUES(5,'{"value":5}'); INSERT INTO t2(id,json) VALUES(6,'{"value":6}'); - SELECT * FROM t1 CROSS JOIN t2 + SELECT *, 'NL' FROM t1 CROSS JOIN t2 WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z WHERE Z.value==t2.id); -} {1 {{"items":[3,5]}} 3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}}} +} {1 {{"items":[3,5]}} 3 {{"value":3}} NL 1 {{"items":[3,5]}} 5 {{"value":5}} NL} do_execsql_test json101-13.110 { - SELECT * FROM t2 CROSS JOIN t1 + SELECT *, 'NL' FROM t2 CROSS JOIN t1 WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z WHERE Z.value==t2.id); -} {3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}} 1 {{"items":[3,5]}}} +} {3 {{"value":3}} 1 {{"items":[3,5]}} NL 5 {{"value":5}} 1 {{"items":[3,5]}} NL} # 2018-05-16 # Incorrect fullkey output from json_each() diff --git a/test/notnull2.test b/test/notnull2.test index 09161efbd..67d7c26a8 100644 --- a/test/notnull2.test +++ b/test/notnull2.test @@ -66,7 +66,7 @@ do_vmstep_test 1.5.2 { SELECT count(*) FROM t2 WHERE EXISTS( SELECT 1 FROM t1 WHERE t1.a=450 AND t2.c IS NULL ) -} +8000 {0} +} 4000 {0} #------------------------------------------------------------------------- reset_db diff --git a/test/parser1.test b/test/parser1.test index ad95b4909..b8d3d8b42 100644 --- a/test/parser1.test +++ b/test/parser1.test @@ -100,4 +100,26 @@ do_execsql_test parser1-3.1 { PRAGMA foreign_key_list(t301); } {0 0 t300 c2 id RESTRICT CASCADE NONE 1 0 t300 c1 id RESTRICT CASCADE NONE} +# 2025-07-01 https://sqlite.org/forum/forumpost/f4878de3e7dd4764 +# Do not allow parse-time optimizations to omit aggregate functions, +# because doing so can change the meaning of the query. +# +unset -nocomplain zero +set zero [expr {0+0}] +do_execsql_test parser1-4.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + SELECT max(x) AND $zero FROM t1; +} 0 +do_execsql_test parser1-4.2 { + SELECT max(x) AND 0 FROM t1; +} 0 +do_execsql_test parser1-4.3 { + SELECT max(x) IN () FROM t1; +} 0 +do_execsql_test parser1-4.4 { + SELECT max(x) NOT IN () FROM t1; +} 1 + + finish_test diff --git a/test/rowvalue.test b/test/rowvalue.test index e2688e903..b6286302d 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -788,6 +788,9 @@ do_execsql_test 33.3 { # INTEGER PRIMARY KEY, and the columns that UNIQUE constraint are # used in a rowvalue-IN operator constraint. # +# 2025-07-07 Discovered that the original fix was incomplete and +# new tests added. See tag-20250707-01 in the code. +# reset_db do_execsql_test 34.1 { CREATE TABLE items ( @@ -804,5 +807,52 @@ do_execsql_test 34.1 { WHERE (Id, Item) IN (SELECT Id, Item FROM items); SELECT Id, Item, test FROM items ORDER BY id; } {1 2 ok 2 2 ok 3 3 ok 4 5 ok} +db null NULL +do_execsql_test 34.2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE INDEX idx ON t1(b,a); + INSERT INTO t1(a,b) VALUES (1, 22); + SELECT * FROM t1 INDEXED BY idx WHERE (b,a) IN (SELECT b,a FROM t1); +} {1 22 NULL NULL} +do_execsql_test 34.3 { + DROP TABLE t1; + CREATE TABLE t1(a, b, c, d); + CREATE INDEX idx ON t1(b,a,a); + INSERT INTO t1(a,b) VALUES (1, 22); + SELECT * FROM t1 INDEXED BY idx WHERE (b,a) IN (SELECT b,a FROM t1); +} {1 22 NULL NULL} +do_execsql_test 34.4 { + DROP TABLE t1; + CREATE TABLE t1(id INTEGER PRIMARY KEY, a INT); + CREATE INDEX t1a ON t1(a,id); -- index includes PRIMARY KEY + CREATE TABLE t2(id INTEGER PRIMARY KEY); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100) + INSERT INTO t1(id,a) SELECT n, 777 FROM c; + INSERT INTO t2 SELECT id FROM t1; + SELECT * + FROM t1 JOIN t2 USING(id) + WHERE t1.a=777 AND t2.id>999 + ORDER BY t1.id; +} {} +do_execsql_test 34.5 { + EXPLAIN QUERY PLAN + SELECT * + FROM t1 JOIN t2 USING(id) + WHERE t1.a=777 AND t2.id>999 + ORDER BY t1.id; +} {/SEARCH t1 USING COVERING INDEX t1a .a=. AND id>../} + +# 2025-08-04 forum post eab63506cf +# +do_execsql_test 35.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(c1 TEXT, c2 INTEGER, PRIMARY KEY(c1, c2)); + INSERT INTO t1(c1, c2) VALUES ('a', 1); + SELECT ('a', 1) IN (SELECT c1, c2 from t1); +} 1 +do_execsql_test 35.1 { + SELECT (1, 'a') IN (SELECT c2, c1 from t1); +} 1 + finish_test diff --git a/test/speedtest1.c b/test/speedtest1.c index 10cd30f1c..968f2cb00 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -159,6 +159,12 @@ static void fatal_error(const char *zMsg, ...){ va_start(ap, zMsg); vfprintf(stderr, zMsg, ap); va_end(ap); +#ifdef SQLITE_SPEEDTEST1_WASM + /* Emscripten complains when exit() is called and anything is left + in the I/O buffers. */ + fflush(stdout); + fflush(stderr); +#endif exit(1); } @@ -2994,7 +3000,13 @@ int main(int argc, char **argv){ /* "mix1" is a macro testset: */ static char zMix1Tests[] = - "main,orm/25,cte/20,json,fp/3,parsenumber/25,rtree/10,star,app"; + "main,orm/25,cte/20,json,fp/3,parsenumber/25,rtree/10,star" +#if !defined(SQLITE_SPEEDTEST1_WASM) + ",app" + /* This test misbehaves in WASM builds: sqlite3_open_v2() is + failing to find the db file for reasons not yet understood. */ +#endif + ; #ifdef SQLITE_SPEEDTEST1_WASM /* Resetting all state is important for the WASM build, which may diff --git a/test/strict1.test b/test/strict1.test index ef2a78fc8..c4c086bdb 100644 --- a/test/strict1.test +++ b/test/strict1.test @@ -212,4 +212,51 @@ do_catchsql_test strict1-9.2.43 { INSERT INTO strict(k) VALUES(43); } {1 {cannot store TEXT value in BLOB column strict.c4}} +do_execsql_test strict1-9.3 { + DROP TABLE strict; + CREATE TABLE strict ( + k INTEGER PRIMARY KEY, + c1 REAL AS(if(k=11,1.5, k=12,2, k=13,'x', k=14,x'34', 0.0)) VIRTUAL, + c2 INT AS(if(k=21,1.5, k=22,2, k=23,'x', k=24,x'34', 0)) VIRTUAL, + c3 TEXT AS(if(k=31,1.5, k=32,2, k=33,'x', k=34,x'34', 'x')) VIRTUAL, + c4 BLOB AS(if(k=41,1.5, k=42,2, k=43,'x', k=44,x'34', x'00')) VIRTUAL, + c5 ANY AS(if(k=51,1.5, k=52,2, k=53,'x', k=54,x'34', 0)) VIRTUAL + ) STRICT; + INSERT INTO strict(k) VALUES(11); + INSERT INTO strict(k) VALUES(12); + INSERT INTO strict(k) VALUES(22); + INSERT INTO strict(k) VALUES(31); + INSERT INTO strict(k) VALUES(32); + INSERT INTO strict(k) VALUES(33); + INSERT INTO strict(k) VALUES(44); + PRAGMA integrity_check; +} {ok} +do_catchsql_test strict1-9.4.13 { + INSERT INTO strict(k) VALUES(13); +} {1 {cannot store TEXT value in REAL column strict.c1}} +do_catchsql_test strict1-9.4.14 { + INSERT INTO strict(k) VALUES(14); +} {1 {cannot store BLOB value in REAL column strict.c1}} +do_catchsql_test strict1-9.4.21 { + INSERT INTO strict(k) VALUES(21); +} {1 {cannot store REAL value in INT column strict.c2}} +do_catchsql_test strict1-9.4.23 { + INSERT INTO strict(k) VALUES(23); +} {1 {cannot store TEXT value in INT column strict.c2}} +do_catchsql_test strict1-9.4.24 { + INSERT INTO strict(k) VALUES(24); +} {1 {cannot store BLOB value in INT column strict.c2}} +do_catchsql_test strict1-9.4.34 { + INSERT INTO strict(k) VALUES(34); +} {1 {cannot store BLOB value in TEXT column strict.c3}} +do_catchsql_test strict1-9.4.41 { + INSERT INTO strict(k) VALUES(41); +} {1 {cannot store REAL value in BLOB column strict.c4}} +do_catchsql_test strict1-9.4.42 { + INSERT INTO strict(k) VALUES(42); +} {1 {cannot store INT value in BLOB column strict.c4}} +do_catchsql_test strict1-9.4.43 { + INSERT INTO strict(k) VALUES(43); +} {1 {cannot store TEXT value in BLOB column strict.c4}} + finish_test diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 0c6982f42..52bc0f34b 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -7,6 +7,18 @@ set testdir [file normalize [file dirname $argv0]] set saved $argv set argv [list] source [file join $testdir testrunner_data.tcl] + +# Estimated amount of work required by displaytype, relative to 'tcl' +# +set estwork(tcl) 1 +set estwork(fuzz) 22 +set estwork(bld) 66 +set estwork(make) 102 + +set estworkfile [file join $testdir testrunner_estwork.tcl] +if {[file readable $estworkfile]} { + source $estworkfile +} source [file join $testdir permutations.test] set argv $saved cd $dir @@ -92,6 +104,7 @@ Usage: $a0 script ?-msvc? CONFIG $a0 status ?-d SECS? ?--cls? $a0 halt + $a0 estwork where SWITCHES are: --buildonly Build test exes but do not run tests @@ -341,6 +354,7 @@ set TRG(schema) { endtime INTEGER, -- End time span INTEGER, -- Total run-time in milliseconds estwork INTEGER, -- Estimated amount of work + estkey TEXT, -- Key used to compute estwork state TEXT CHECK( state IN ('','ready','running','done','failed','omit','halt') ), ntest INT, -- Number of test cases run nerr INT, -- Number of errors reported @@ -359,13 +373,6 @@ set TRG(schema) { } #------------------------------------------------------------------------- -# Estimated amount of work required by displaytype, relative to 'tcl' -# -set estwork(tcl) 1 -set estwork(fuzz) 11 -set estwork(bld) 56 -set estwork(make) 97 - #-------------------------------------------------------------------------- # Check if this script is being invoked to run a single file. If so, # run it. @@ -453,6 +460,59 @@ if {[llength $argv]==1 #-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Check if this is the "estwork" command: +# +# Generate (on standard output) a set of estwork() values based on the lastest +# test case, that can be used to replace the test/testrunner_estwork.tcl file. +# +if {[llength $argv]==1 + && [string compare -nocase estwork [lindex $argv 0]]==0 +} { + sqlite3 mydb $TRG(dbname) + set njob [mydb one {SELECT count(*) FROM jobs WHERE state='done'}] + if {$njob<1000} { + puts "Too few completed jobs to do a work estimate." + puts "Have $njob but not need at least 1000." + mydb close + exit 1 + } + set badjobs [mydb one {SELECT count(*) FROM jobs WHERE state<>'done'}] + if {$badjobs} { + puts "Database contains $badjobs incomplete jobs." + mydb close + exit 1 + } + set half [mydb one {SELECT count(*)/2 FROM jobs WHERE displaytype='tcl'}] + set scale [mydb one {SELECT span FROM jobs WHERE displaytype='tcl' + ORDER BY span LIMIT 1 OFFSET $half}] + mydb eval { + SELECT estkey, CAST(avg(span)/$scale AS INT) AS cost + FROM jobs + GROUP BY estkey + HAVING cost>=2 + } { + set estwork($estkey) $cost + } + set avgtcl [mydb one {SELECT avg(span) FROM jobs WHERE displaytype='tcl'}] + set estwork(tcl) 1 + foreach type {bld fuzz make} { + set avg [mydb one {SELECT avg(span) FROM jobs WHERE displaytype=$type}] + if {$avg!=""} { + set estwork($type) [expr {int($avg/$avgtcl)}] + } + } + mydb close + puts "# Estimated relative cost of various jobs, based on the \"estkey\" field." + puts "# Computed by the \"test/testrunner.tcl estwork\" command." + puts "#" + foreach key [lsort [array names estwork]] { + puts "set [list estwork($key)] $estwork($key)" + } + exit +} +#-------------------------------------------------------------------------- + +#-------------------------------------------------------------------------- # Check if this is the "help" command: # if {[string compare -nocase help [lindex $argv 0]]==0} { @@ -562,8 +622,8 @@ proc show_status {db cls} { set srcdir [file dirname [file dirname $TRG(info_script)]] set line "Running: $S(running) (max: $nJob)" - if {$S(running)>0 && $fin>10} { - set tmleft [expr {($tm/$fin)*($totalw-$fin)}] + if {$S(running)>0 && [set pct [expr {int(($fin*100.0)/$totalw)}]]>=4} { + set tmleft [expr {($tm/double($fin))*($totalw-$fin)}] if {$tmleft<0.02*$tm} { set tmleft [expr {$tm*0.02}] } @@ -571,6 +631,7 @@ proc show_status {db cls} { if {[string length $line]+[string length $etc]<80} { append line $etc } + # append line " $pct%" } puts [format %-79.79s $line] if {$S(running)>0} { @@ -985,12 +1046,31 @@ proc add_job {args} { set state "" if {$A(-depid)==""} { set state ready } set type $A(-displaytype) - set ew $estwork($type) + set displayname $A(-displayname) + switch $type { + tcl { + set ek [file tail [lindex $displayname end]] + } + bld { + set ek [lindex $displayname end] + } + fuzz { + set ek [lrange $displayname 1 2] + } + make { + set ek [lindex $displayname end] + } + } + if {[info exists estwork($ek)]} { + set ew $estwork($ek) + } else { + set ew $estwork($type) + } trdb eval { INSERT INTO jobs( - displaytype, displayname, build, dirname, cmd, depid, priority, estwork, - state + displaytype, displayname, build, dirname, cmd, depid, priority, + estwork, estkey, state ) VALUES ( $type, $A(-displayname), @@ -1000,6 +1080,7 @@ proc add_job {args} { $A(-depid), $A(-priority), $ew, + $ek, $state ) } @@ -1586,12 +1667,13 @@ proc progress_report {} { } } set report "[elapsetime $tmms] [join $text { }]" - if {$wdone>0} { - set tmleft [expr {($tmms/$wdone)*($wtotal-$wdone)}] - set etc " ETC [elapsetime $tmleft]" + if {$wdone>0 && [set pct [expr {int(($wdone*100.0)/$wtotal)}]]>=4} { + set tmleft [expr {($tmms/double($wdone))*($wtotal-$wdone)}] + set etc " ETC [elapsetime $tmleft]" if {[string length $report]+[string length $etc]<80} { append report $etc } + # append report " $pct%" } puts -nonewline [format %-79.79s $report]\r flush stdout diff --git a/test/testrunner_estwork.tcl b/test/testrunner_estwork.tcl new file mode 100644 index 000000000..c139394a5 --- /dev/null +++ b/test/testrunner_estwork.tcl @@ -0,0 +1,488 @@ +# Estimated relative cost of various jobs, based on the "estkey" field. +# Computed by the "test/testrunner.tcl estwork" command. +# +set estwork((fuzzcheck)) 1291 +set estwork((fuzzcheck-asan)) 2427 +set estwork((fuzzcheck-ubsan)) 2749 +set estwork((sessionfuzz)) 1276 +set estwork((sqlite3)) 1281 +set estwork((testfixture)) 1546 +set estwork(aggerror.test) 5 +set estwork(alter.test) 20 +set estwork(alter3.test) 2 +set estwork(alterauth.test) 5 +set estwork(altercol.test) 5 +set estwork(alterdropcol.test) 17 +set estwork(alterlegacy.test) 2 +set estwork(altertab.test) 34 +set estwork(altertab2.test) 2 +set estwork(altertab3.test) 12 +set estwork(amatch1.test) 40 +set estwork(atof1.test) 146 +set estwork(atomic2.test) 2 +set estwork(auth.test) 2 +set estwork(autoindex1.test) 5 +set estwork(autoindex3.test) 5 +set estwork(autovacuum.test) 15 +set estwork(avfs.test) 16 +set estwork(avtrans.test) 285 +set estwork(backup2.test) 15 +set estwork(bestindex4.test) 12 +set estwork(bigrow.test) 5 +set estwork(bitvec.test) 94 +set estwork(bld) 99 +set estwork(blob.test) 2 +set estwork(boundary1.test) 8 +set estwork(boundary2.test) 16 +set estwork(boundary3.test) 14 +set estwork(btree01.test) 20 +set estwork(busy.test) 2 +set estwork(busy2.test) 288 +set estwork(capi3.test) 5 +set estwork(capi3b.test) 6 +set estwork(capi3c.test) 5 +set estwork(capi3d.test) 8 +set estwork(capi3e.test) 20 +set estwork(changes.test) 11 +set estwork(chunksize.test) 5 +set estwork(closure01.test) 63 +set estwork(collate5.test) 2 +set estwork(conflict.test) 8 +set estwork(conflict2.test) 4 +set estwork(conflict3.test) 8 +set estwork(corrupt2.test) 2 +set estwork(corrupt4.test) 91 +set estwork(corrupt7.test) 5 +set estwork(corrupt8.test) 3 +set estwork(corruptB.test) 4 +set estwork(corruptF.test) 6 +set estwork(count.test) 18 +set estwork(crash8.test) 14 +set estwork(createtab.test) 3 +set estwork(csv01.test) 7 +set estwork(cursorhint.test) 4 +set estwork(date.test) 17 +set estwork(date4.test) 64 +set estwork(date5.test) 3 +set estwork(dbdata.test) 77 +set estwork(dbfuzz001.test) 2 +set estwork(dbstatus.test) 4 +set estwork(decimal.test) 2 +set estwork(delete.test) 4 +set estwork(diskfull.test) 34 +set estwork(e_blobbytes.test) 11 +set estwork(e_createtable.test) 14 +set estwork(e_delete.test) 3 +set estwork(e_droptrigger.test) 6 +set estwork(e_dropview.test) 2 +set estwork(e_expr.test) 251 +set estwork(e_select.test) 5 +set estwork(e_select2.test) 5 +set estwork(e_vacuum.test) 6 +set estwork(e_walauto.test) 51 +set estwork(e_walckpt.test) 6 +set estwork(enc.test) 2 +set estwork(enc3.test) 11 +set estwork(enc4.test) 3 +set estwork(eval.test) 2 +set estwork(exists.test) 4 +set estwork(existsexpr.test) 44 +set estwork(expert1.test) 14 +set estwork(expr.test) 5 +set estwork(filectrl.test) 5 +set estwork(filefmt.test) 6 +set estwork(fkey2.test) 9 +set estwork(fpconv1.test) 40 +set estwork(fts3aa.test) 18 +set estwork(fts3ad.test) 10 +set estwork(fts3ag.test) 6 +set estwork(fts3aj.test) 8 +set estwork(fts3am.test) 5 +set estwork(fts3auto.test) 29 +set estwork(fts3b.test) 41 +set estwork(fts3c.test) 22 +set estwork(fts3conf.test) 3 +set estwork(fts3corrupt4.test) 7 +set estwork(fts3corrupt5.test) 8 +set estwork(fts3corrupt6.test) 13 +set estwork(fts3d.test) 2 +set estwork(fts3defer2.test) 2 +set estwork(fts3expr.test) 10 +set estwork(fts3expr2.test) 8 +set estwork(fts3expr3.test) 145 +set estwork(fts3f.test) 9 +set estwork(fts3first.test) 3 +set estwork(fts3matchinfo.test) 4 +set estwork(fts3misc.test) 54 +set estwork(fts3prefix.test) 4 +set estwork(fts3prefix2.test) 8 +set estwork(fts3query.test) 23 +set estwork(fts3varint.test) 2 +set estwork(fts4aa.test) 34 +set estwork(fts4content.test) 5 +set estwork(fts4incr.test) 19 +set estwork(fts4noti.test) 3 +set estwork(fts4opt.test) 129 +set estwork(fts4unicode.test) 45 +set estwork(fts5aa.test) 442 +set estwork(fts5ab.test) 54 +set estwork(fts5ad.test) 33 +set estwork(fts5ae.test) 3 +set estwork(fts5af.test) 13 +set estwork(fts5ag.test) 13 +set estwork(fts5ah.test) 265 +set estwork(fts5al.test) 2 +set estwork(fts5auto.test) 92 +set estwork(fts5connect.test) 5 +set estwork(fts5content.test) 9 +set estwork(fts5contentless.test) 23 +set estwork(fts5contentless2.test) 189 +set estwork(fts5contentless4.test) 234 +set estwork(fts5contentless5.test) 9 +set estwork(fts5delete.test) 62 +set estwork(fts5doclist.test) 11 +set estwork(fts5expr.test) 10 +set estwork(fts5full.test) 42 +set estwork(fts5hash.test) 17 +set estwork(fts5integrity.test) 84 +set estwork(fts5interrupt.test) 16 +set estwork(fts5locale.test) 5 +set estwork(fts5matchinfo.test) 11 +set estwork(fts5merge.test) 29 +set estwork(fts5merge2.test) 9 +set estwork(fts5misc.test) 8 +set estwork(fts5multiclient.test) 2 +set estwork(fts5optimize.test) 13 +set estwork(fts5optimize2.test) 737 +set estwork(fts5optimize3.test) 280 +set estwork(fts5origintext.test) 144 +set estwork(fts5origintext3.test) 8 +set estwork(fts5origintext4.test) 30 +set estwork(fts5origintext5.test) 462 +set estwork(fts5origintext6.test) 55 +set estwork(fts5porter.test) 57 +set estwork(fts5query.test) 8 +set estwork(fts5restart.test) 10 +set estwork(fts5rowid.test) 35 +set estwork(fts5secure.test) 61 +set estwork(fts5secure3.test) 796 +set estwork(fts5secure4.test) 141 +set estwork(fts5secure5.test) 29 +set estwork(fts5secure6.test) 33 +set estwork(fts5secure7.test) 680 +set estwork(fts5simple.test) 7 +set estwork(fts5simple2.test) 2 +set estwork(fts5synonym.test) 13 +set estwork(fts5synonym2.test) 384 +set estwork(fts5tok2.test) 92 +set estwork(fts5tokenizer.test) 2 +set estwork(fts5tokenizer3.test) 8 +set estwork(fts5trigram.test) 2 +set estwork(fts5unicode2.test) 28 +set estwork(fts5unicode3.test) 72 +set estwork(fts5unindexed.test) 7 +set estwork(fts5update.test) 161 +set estwork(fts5update2.test) 11 +set estwork(fts5vocab.test) 11 +set estwork(fts5vocab2.test) 15 +set estwork(func.test) 36 +set estwork(fuzz) 68 +set estwork(fuzz-oss1.test) 11 +set {estwork(fuzzcheck --slice)} 499 +set {estwork(fuzzcheck fuzzdata1.db)} 301 +set {estwork(fuzzcheck fuzzdata2.db)} 275 +set {estwork(fuzzcheck fuzzdata3.db)} 72 +set {estwork(fuzzcheck fuzzdata4.db)} 30 +set {estwork(fuzzcheck fuzzdata5.db)} 299 +set {estwork(fuzzcheck fuzzdata6.db)} 91 +set {estwork(fuzzcheck fuzzdata7.db)} 201 +set {estwork(fuzzcheck fuzzdata8.db)} 331 +set {estwork(fuzzcheck-asan --slice)} 2037 +set {estwork(fuzzcheck-asan fuzzdata1.db)} 3112 +set {estwork(fuzzcheck-asan fuzzdata2.db)} 8753 +set {estwork(fuzzcheck-asan fuzzdata3.db)} 459 +set {estwork(fuzzcheck-asan fuzzdata4.db)} 123 +set {estwork(fuzzcheck-asan fuzzdata5.db)} 1178 +set {estwork(fuzzcheck-asan fuzzdata6.db)} 1356 +set {estwork(fuzzcheck-asan fuzzdata7.db)} 938 +set {estwork(fuzzcheck-asan fuzzdata8.db)} 1451 +set {estwork(fuzzcheck-ubsan --slice)} 1729 +set {estwork(fuzzcheck-ubsan fuzzdata1.db)} 1180 +set {estwork(fuzzcheck-ubsan fuzzdata2.db)} 876 +set {estwork(fuzzcheck-ubsan fuzzdata3.db)} 306 +set {estwork(fuzzcheck-ubsan fuzzdata4.db)} 95 +set {estwork(fuzzcheck-ubsan fuzzdata5.db)} 1356 +set {estwork(fuzzcheck-ubsan fuzzdata6.db)} 333 +set {estwork(fuzzcheck-ubsan fuzzdata7.db)} 883 +set {estwork(fuzzcheck-ubsan fuzzdata8.db)} 1124 +set estwork(fuzzer1.test) 6 +set estwork(gencol1.test) 3 +set estwork(hook.test) 2 +set estwork(in4.test) 4 +set estwork(in7.test) 6 +set estwork(incrblob2.test) 2 +set estwork(incrblob3.test) 5 +set estwork(incrvacuum.test) 10 +set estwork(incrvacuum2.test) 17 +set estwork(incrvacuum3.test) 17 +set estwork(index.test) 3 +set estwork(index2.test) 8 +set estwork(index4.test) 33 +set estwork(index5.test) 99 +set estwork(index6.test) 2 +set estwork(indexexpr1.test) 2 +set estwork(insert3.test) 5 +set estwork(insert4.test) 2 +set estwork(intarray.test) 8 +set estwork(intck1.test) 34 +set estwork(intck2.test) 37 +set estwork(interrupt.test) 28 +set estwork(io.test) 3 +set estwork(join.test) 3 +set estwork(join3.test) 6 +set estwork(join5.test) 20 +set estwork(join7.test) 2 +set estwork(join8.test) 3 +set estwork(join9.test) 2 +set estwork(joinA.test) 11 +set estwork(joinB.test) 7 +set estwork(joinC.test) 6 +set estwork(joinD.test) 168 +set estwork(json103.test) 6 +set estwork(json106.test) 690 +set estwork(keyword1.test) 3 +set estwork(like2.test) 2 +set estwork(like3.test) 2 +set estwork(limit.test) 3 +set estwork(literal.test) 6 +set estwork(lock.test) 39 +set estwork(lock4.test) 2 +set estwork(lock5.test) 4 +set estwork(make) 102 +set estwork(manydb.test) 12 +set estwork(mem5.test) 2 +set estwork(memdb.test) 9 +set estwork(memdb1.test) 2 +set estwork(memjournal2.test) 164 +set estwork(memsubsys1.test) 8 +set estwork(misc1.test) 6 +set estwork(misc2.test) 6 +set estwork(misc5.test) 11 +set estwork(misc8.test) 5 +set estwork(mmap1.test) 8 +set estwork(mmap2.test) 10 +set estwork(mmapwarm.test) 2 +set estwork(multiplex.test) 30 +set estwork(multiplex2.test) 7 +set estwork(nan.test) 2 +set estwork(notify3.test) 5 +set estwork(orderby1.test) 11 +set estwork(orderby2.test) 2 +set estwork(orderby5.test) 11 +set estwork(orderby6.test) 6 +set estwork(orderby8.test) 20 +set estwork(orderbyA.test) 2 +set estwork(oserror.test) 7 +set estwork(ovfl.test) 11 +set estwork(pager1.test) 81 +set estwork(pager2.test) 129 +set estwork(pagesize.test) 3 +set estwork(percentile.test) 86 +set estwork(pragma.test) 4 +set estwork(pragma4.test) 18 +set estwork(printf.test) 21 +set estwork(printf2.test) 5 +set estwork(quota.test) 14 +set estwork(randexpr1.test) 35 +set estwork(rbu10.test) 16 +set estwork(rbu13.test) 5 +set estwork(rbuexlock.test) 5 +set estwork(rbumisc.test) 2 +set estwork(rbutemplimit.test) 2 +set estwork(readonly.test) 95 +set estwork(recover.test) 3 +set estwork(recover1.test) 7 +set estwork(recovercorrupt3.test) 5 +set estwork(recoverold.test) 7 +set estwork(recoverpgsz.test) 6 +set estwork(recoverrowid.test) 3 +set estwork(returning1.test) 6 +set estwork(rollback2.test) 2 +set estwork(round1.test) 261 +set estwork(rowhash.test) 85 +set estwork(rowid.test) 3 +set estwork(rowvalue.test) 2 +set estwork(rowvalue2.test) 38 +set estwork(rowvalue4.test) 2 +set estwork(rowvalueA.test) 2 +set estwork(rowvaluevtab.test) 2 +set estwork(rtree1.test) 4 +set estwork(rtree2.test) 758 +set estwork(rtree6.test) 6 +set estwork(rtree8.test) 15 +set estwork(rtree9.test) 15 +set estwork(rtreeA.test) 7 +set estwork(rtreeB.test) 7 +set estwork(rtreeE.test) 47 +set estwork(rtreeH.test) 24 +set estwork(rtreecheck.test) 2 +set estwork(rtreedoc.test) 8 +set estwork(rtreedoc3.test) 76 +set estwork(savepoint.test) 10 +set estwork(savepoint2.test) 57 +set estwork(schema2.test) 15 +set estwork(schema3.test) 2 +set estwork(schema5.test) 5 +set estwork(select1.test) 2 +set estwork(select2.test) 17 +set estwork(select3.test) 2 +set estwork(selectA.test) 2 +set estwork(selectB.test) 2 +set estwork(selectG.test) 30 +set estwork(session1.test) 5 +set estwork(session2.test) 33 +set estwork(session5.test) 63 +set estwork(session9.test) 3 +set estwork(sessionG.test) 60 +set estwork(sessionH.test) 18 +set estwork(sessionalter.test) 3 +set estwork(sessionat.test) 2 +set estwork(sessionblob.test) 3 +set {estwork(sessionfuzz sessionfuzz-data1.db)} 5 +set estwork(sessioninvert.test) 2 +set estwork(sessionnoop.test) 2 +set estwork(sessionnoop2.test) 8 +set estwork(sessionrebase.test) 8 +set estwork(shared.test) 7 +set estwork(sharedA.test) 48 +set estwork(shell1.test) 36 +set estwork(shell2.test) 11 +set estwork(shell3.test) 3 +set estwork(shell4.test) 2 +set estwork(shell5.test) 16 +set estwork(shell6.test) 3 +set estwork(shell8.test) 104 +set estwork(shell9.test) 3 +set estwork(shellA.test) 2 +set estwork(shmlock.test) 27 +set estwork(sidedelete.test) 10 +set estwork(skipscan1.test) 7 +set estwork(skipscan2.test) 5 +set estwork(sort.test) 38 +set estwork(sort2.test) 540 +set estwork(sort5.test) 16 +set estwork(spellfix.test) 5 +set estwork(spellfix2.test) 5 +set estwork(spellfix4.test) 11 +set estwork(starschema1.test) 2 +set estwork(strict1.test) 5 +set estwork(swarmvtab.test) 110 +set estwork(swarmvtab3.test) 14 +set estwork(syscall.test) 4 +set estwork(table.test) 62 +set estwork(tableapi.test) 8 +set estwork(tcl) 1 +set estwork(tclsqlite.test) 29 +set estwork(temptable2.test) 274 +set estwork(thread3.test) 21 +set estwork(timediff1.test) 5 +set estwork(tkt-2d1a5c67d.test) 6 +set estwork(tkt-38cb5df375.test) 2 +set estwork(tkt-4dd95f6943.test) 2 +set estwork(tkt-5e10420e8d.test) 5 +set estwork(tkt-6bfb98dfc0.test) 5 +set estwork(tkt-80e031a00f.test) 25 +set estwork(tkt-9d68c883.test) 3 +set estwork(tkt-9f2eb3abac.test) 5 +set estwork(tkt-b1d3a2e531.test) 5 +set estwork(tkt-b72787b1.test) 5 +set estwork(tkt-b75a9ca6b0.test) 8 +set estwork(tkt-d11f09d36e.test) 9 +set estwork(tkt-fc62af4523.test) 10 +set estwork(tkt1435.test) 7 +set estwork(tkt1644.test) 5 +set estwork(tkt1667.test) 20 +set estwork(tkt1873.test) 2 +set estwork(tkt2192.test) 3 +set estwork(tkt2285.test) 19 +set estwork(tkt2332.test) 2 +set estwork(tkt2409.test) 24 +set estwork(tkt3334.test) 5 +set estwork(tkt3357.test) 10 +set estwork(tkt3630.test) 2 +set estwork(tkt3832.test) 8 +set estwork(tkt3838.test) 5 +set estwork(tkt3918.test) 3 +set estwork(tkt4018.test) 48 +set estwork(tokenize.test) 9 +set estwork(tpch01.test) 6 +set estwork(trans.test) 258 +set estwork(trigger2.test) 15 +set estwork(trigger5.test) 2 +set estwork(trigger8.test) 30 +set estwork(triggerA.test) 36 +set estwork(triggerB.test) 4 +set estwork(triggerC.test) 67 +set estwork(triggerD.test) 3 +set estwork(types.test) 2 +set estwork(types2.test) 2 +set estwork(unionall2.test) 9 +set estwork(unionvtab.test) 2 +set estwork(update.test) 7 +set estwork(upsert3.test) 6 +set estwork(upsert5.test) 5 +set estwork(vacuum.test) 2 +set estwork(vacuum5.test) 3 +set estwork(vacuum6.test) 174 +set estwork(vacuummem.test) 132 +set estwork(varint.test) 8 +set estwork(view.test) 2 +set estwork(vtab1.test) 13 +set estwork(vtab6.test) 12 +set estwork(vtabC.test) 33 +set estwork(vtabD.test) 56 +set estwork(vtab_alter.test) 6 +set estwork(wal.test) 38 +set estwork(wal2.test) 7 +set estwork(wal3.test) 196 +set estwork(wal4.test) 70 +set estwork(wal5.test) 22 +set estwork(wal64k.test) 20 +set estwork(wal7.test) 3 +set estwork(wal9.test) 24 +set estwork(walbak.test) 2 +set estwork(walcksum.test) 3 +set estwork(walcrash4.test) 59 +set estwork(waloverwrite.test) 3 +set estwork(walpersist.test) 6 +set estwork(walprotocol2.test) 2 +set estwork(walro2.test) 11 +set estwork(walsetlk.test) 1610 +set estwork(walsetlk3.test) 3 +set estwork(walsetlk_recover.test) 193 +set estwork(walshared.test) 5 +set estwork(walvfs.test) 476 +set estwork(where.test) 24 +set estwork(where3.test) 4 +set estwork(where6.test) 2 +set estwork(where7.test) 20 +set estwork(where8.test) 95 +set estwork(where9.test) 16 +set estwork(whereB.test) 5 +set estwork(whereE.test) 6 +set estwork(whereN.test) 2 +set estwork(window1.test) 5 +set estwork(window2.test) 8 +set estwork(window3.test) 89 +set estwork(window4.test) 3 +set estwork(window5.test) 2 +set estwork(window8.test) 33 +set estwork(windowA.test) 5 +set estwork(windowD.test) 6 +set estwork(with1.test) 114 +set estwork(with2.test) 4 +set estwork(with5.test) 2 +set estwork(withM.test) 4 +set estwork(without_rowid1.test) 2 +set estwork(without_rowid3.test) 9 +set estwork(without_rowid4.test) 6 diff --git a/test/vtabH.test b/test/vtabH.test index 07704cefb..1496f49b5 100644 --- a/test/vtabH.test +++ b/test/vtabH.test @@ -190,10 +190,10 @@ if {$tcl_platform(platform) ne "windows" || \ lappend res "/$p" } } - set num_root_files [llength $root_files] + set num_root_files [llength $res] do_test 3.1 { sort_files [execsql { - SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $num_root_files; + SELECT path FROM fstree WHERE path NOT GLOB '*$*' LIMIT $num_root_files }] true } [sort_files $res true] diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index b750593c9..2d7ea84e5 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -62,10 +62,6 @@ cp $TOP/main.mk $TMPSPACE cd $TMPSPACE -# Clean up emacs-generated backup files from the target -rm -f ./autosetup/*~ ./autosetup/teaish/*~ -rm -f ./*~ - #if true; then # Clean up *~ files (emacs-generated backups). # This bit is only for use during development of @@ -83,8 +79,12 @@ cat <<EOF > tea/generic/tclsqlite3.c EOF cat $TOP/src/tclsqlite.c >> tea/generic/tclsqlite3.c -find . -type f -name '*~' -exec rm -f \{} \; -find . -type f -name '#*#' -exec rm -f \{} \; +# Clean up some local remnants from the tarball. +rm -f tea/.env-* # autosetup environment overrides +find . -type f -name '*~' -exec rm -f \{} \; # backup files +find . -type f -name '#*#' -exec rm -f \{} \; # emacs lock files +find . -type f -name '*.o' -exec rm -f \{} \; +find . -type f -name '*.so' -exec rm -f \{} \; ./configure && make dist tar xzf sqlite-$VERSION.tar.gz diff --git a/tool/mktoolzip.tcl b/tool/mktoolzip.tcl index c22318441..041bf28cd 100644 --- a/tool/mktoolzip.tcl +++ b/tool/mktoolzip.tcl @@ -12,9 +12,27 @@ # sqlite3_analyzer -- Space analyzer # sqlite3_rsync -- Remote db sync # +# On Windows, add: +# +# sqlite3.def +# sqlite3.dll +# +# Add the --snapshot option to generate a snapshot ZIP archive instead of +# a release ZIP archive. +# +set bSnapshot 0 +for {set i 0} {$i<[llength $argv]} {incr i} { + set a [lindex $argv $i] + if {$a eq "-snapshot" || $a eq "--snapshot"} { + set bSnapshot 1 + continue + } + puts stderr "unknown argument: $a" + exit 1 +} switch $tcl_platform(os) { {Windows NT} { - set OS win32 + set OS win set EXE .exe } Linux { @@ -49,13 +67,39 @@ switch $tcl_platform(machine) { set ARCH unk } } -set in [open [file join [file dirname [file dirname [info script]]] VERSION]] -set vers [read $in] -close $in -scan $vers %d.%d.%d v1 v2 v3 -set v2 [format 3%02d%02d00 $v2 $v3] +if {$bSnapshot} { + set in [open [file join [file dirname [file dirname [info script]]] manifest]] + set manifest [read $in] + close $in + regexp {\nD (.{16})} $manifest all date + regsub -all {[-:T]} $date {} v2 +} else { + set in [open [file join [file dirname [file dirname [info script]]] VERSION]] + set vers [read $in] + close $in + scan $vers %d.%d.%d v1 v2 v3 + set v2 [format 3%02d%02d00 $v2 $v3] +} + set name sqlite-tools-$OS-$ARCH-$v2.zip -set toollist "sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE sqlite3_rsync$EXE" -puts "zip $name {*}$toollist" -exec zip $name {*}$toollist -puts "$name: [file size $name] bytes" +set filelist "sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE sqlite3_rsync$EXE" +proc make_zip_archive {name filelist} { + file delete -force $name + puts "fossil test-filezip $name $filelist" + if {[catch {exec fossil test-filezip $name {*}$filelist}]} { + puts "^--- Unable. Trying again as:" + puts "zip $name $filelist" + file delete -force $name + exec zip $name {*}$filelist + } + puts "$name: [file size $name] bytes" +} +make_zip_archive $name $filelist + +# On Windows, also try to construct a DLL +# +if {$OS eq "win" && [file exists sqlite3.dll] && [file exists sqlite3.def]} { + set name sqlite-dll-win-$ARCH-$v2.zip + set filelist [list sqlite3.def sqlite3.dll] + make_zip_archive $name $filelist +} |