Fix some problems with OOM handling in vdbesort.c.

FossilOrigin-Name: 47e702bd8392bc50c4edaf6a2c8c499af87b520e

4 files changed, 48 insertions, 29 deletions

diff --git a/manifest b/manifest
index 8bc41fb37..2cae57fb9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sname\sof\sthe\sSorterThread\sobject\sto\s"SortSubtask"\sto\savoid\sconfusion\nwith\sthe\sSQLiteThread\sobject.
-D 2014-04-02T14:38:14.734
+C Fix\ssome\sproblems\swith\sOOM\shandling\sin\svdbesort.c.
+D 2014-04-02T15:15:25.762
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4
 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea
 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa
 F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447
-F src/vdbesort.c 0cb83fc36f8a56cc6e4ecdefc3c7d4155169cef2
+F src/vdbesort.c e830ea4a7333ff07177fc367918ede2b33fcfe10
 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767
 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
 F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8
@@ -671,7 +671,7 @@ F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151
 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a
 F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d
 F test/malloc9.test 2307c6ee3703b0a21391f3ea92388b4b73f9105e
-F test/mallocA.test 1ba0367fb5434e7bc2fa4afcb30b14174d91b160
+F test/mallocA.test c049224adeb0244b8f6eb770c1fa6ac40f9b3518
 F test/mallocAll.test 98f1be74bc9f49a858bc4f361fc58e26486798be
 F test/mallocB.test bc475ab850cda896142ab935bbfbc74c24e51ed6
 F test/mallocC.test 3dffe16532f109293ce1ccecd0c31dca55ef08c4
@@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 821d1ac4504243fa13b9e3c0d56361ad9fb80d78
-R 2283517ddbfd0881db16b2410c51a15c
-U drh
-Z e6ea0be46d70fe73f5ea59fa8b577076
+P 4ee2d910fbbed8d4def15e4e99ee225839f3a739
+R 18420994a2ccade11be299f196096342
+U dan
+Z 3f3c411cd945b7957dc8b5eeb10e94fa
diff --git a/manifest.uuid b/manifest.uuid
index 0fe5750cf..4eba9e128 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4ee2d910fbbed8d4def15e4e99ee225839f3a739
\ No newline at end of file
+47e702bd8392bc50c4edaf6a2c8c499af87b520e
\ No newline at end of file
diff --git a/src/vdbesort.c b/src/vdbesort.c
index ff315494e..50680007b 100644
--- a/src/vdbesort.c
+++ b/src/vdbesort.c
@@ -443,10 +443,13 @@ static int vdbeSorterIterInit(
   pIter->iReadOff = iStart;
   pIter->nAlloc = 128;
   pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc);
-
-  /* Try to xFetch() a mapping of the entire temp file. If this is possible,
-  ** the PMA will be read via the mapping. Otherwise, use xRead().  */
-  rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap);
+  if( pIter->aAlloc ){
+    /* Try to xFetch() a mapping of the entire temp file. If this is possible,
+    ** the PMA will be read via the mapping. Otherwise, use xRead().  */
+    rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap);
+  }else{
+    rc = SQLITE_NOMEM;
+  }
 
   if( rc==SQLITE_OK ){
     if( pMap ){
@@ -698,23 +701,15 @@ static SorterMerger *vdbeSorterMergerNew(int nIter){
 }
 
 /*
-** Reset a merger
+** Free the SorterMerger object passed as the only argument.
 */
-static void vdbeSorterMergerReset(SorterMerger *pMerger){
+static void vdbeSorterMergerFree(SorterMerger *pMerger){
   int i;
   if( pMerger ){
     for(i=0; i<pMerger->nTree; i++){
       vdbeSorterIterZero(&pMerger->aIter[i]);
     }
   }
-}
-
-
-/*
-** Free the SorterMerger object passed as the only argument.
-*/
-static void vdbeSorterMergerFree(SorterMerger *pMerger){
-  vdbeSorterMergerReset(pMerger);
   sqlite3_free(pMerger);
 }
 
@@ -724,6 +719,8 @@ static void vdbeSorterMergerFree(SorterMerger *pMerger){
 void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){
   int i;
   vdbeSorterJoinAll(pSorter, SQLITE_OK);
+  vdbeSorterMergerFree(pSorter->pMerger);
+  pSorter->pMerger = 0;
   for(i=0; i<pSorter->nThread; i++){
     SortSubtask *pThread = &pSorter->aThread[i];
     vdbeSortSubtaskCleanup(db, pThread);
@@ -731,7 +728,6 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){
   if( pSorter->aMemory==0 ){
     vdbeSorterRecordFree(0, pSorter->pRecord);
   }
-  vdbeSorterMergerReset(pSorter->pMerger);
   pSorter->pRecord = 0;
   pSorter->nInMemory = 0;
   pSorter->bUsePMA = 0;
@@ -1292,11 +1288,13 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){
 #endif
     {
       /* Use the foreground thread for this operation */
-      u8 *aMem;
       rc = vdbeSorterRunThread(pThread);
-      aMem = pThread->aListMemory;
-      pThread->aListMemory = pSorter->aMemory;
-      pSorter->aMemory = aMem;
+      if( rc==SQLITE_OK ){
+        u8 *aMem = pThread->aListMemory;
+        pThread->aListMemory = pSorter->aMemory;
+        pSorter->aMemory = aMem;
+        assert( pThread->pList==0 );
+      }
     }
   }
 
diff --git a/test/mallocA.test b/test/mallocA.test
index 61e88a61e..d6d6de822 100644
--- a/test/mallocA.test
+++ b/test/mallocA.test
@@ -25,7 +25,6 @@ if {!$MEMDEBUG} {
    return
 }
 
-
 # Construct a test database
 #
 forcedelete test.db.bu
@@ -116,6 +115,28 @@ ifcapable stat3 {
   }
 }
 
+do_execsql_test 7.0 {
+  PRAGMA cache_size = 5;
+}
+do_faultsim_test 7 -faults oom-trans* -prep {
+  if {$iFail < 500} { set iFail 2000 }
+  if {$iFail > 1215} { set iFail 2000 }
+} -body {
+  execsql {
+    WITH r(x,y) AS (
+      SELECT 1, randomblob(100)
+      UNION ALL
+      SELECT x+1, randomblob(100) FROM r
+      LIMIT 1000
+    )
+    SELECT count(x), length(y) FROM r GROUP BY (x%5)
+  }
+} -test {
+  set res [list 200 100 200 100 200 100 200 100 200 100]
+  faultsim_test_result [list 0 $res]
+}
+
+
 # Ensure that no file descriptors were leaked.
 do_test malloc-99.X {
   catch {db close}