Attempt to reduce the memory used by VALUES clauses in as many statements as possible, not just INSERT. This branch still has problems.

FossilOrigin-Name: 17d1f7cfabc7593d0725051b0c7c9619a23a482265f30f15ab9493fef5caeeb0

6 files changed, 140 insertions, 32 deletions

diff --git a/src/expr.c b/src/expr.c
index 6640d1907..e4bfa995d 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1869,6 +1869,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){
     pNewItem->iCursor = pOldItem->iCursor;
     pNewItem->addrFillSub = pOldItem->addrFillSub;
     pNewItem->regReturn = pOldItem->regReturn;
+    pNewItem->regResult = pOldItem->regResult;
     if( pNewItem->fg.isIndexedBy ){
       pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy);
     }
diff --git a/src/insert.c b/src/insert.c
index 095298b90..7b253e926 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -577,6 +577,104 @@ void sqlite3AutoincrementEnd(Parse *pParse){
 # define autoIncStep(A,B,C)
 #endif /* SQLITE_OMIT_AUTOINCREMENT */
 
+void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){
+  if( pVal->pSrc->nSrc>0 ){
+    SrcItem *pItem = &pVal->pSrc->a[0];
+    sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->regReturn);
+    sqlite3VdbeJumpHere(pParse->pVdbe, pItem->addrFillSub - 1);
+  }
+}
+
+static int multiValueIsConstant(ExprList *pRow){
+  int ii;
+  for(ii=0; ii<pRow->nExpr; ii++){
+    if( 0==sqlite3ExprIsConstant(pRow->a[ii].pExpr) ) return 0;
+  }
+  return 1;
+}
+
+Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){
+  SrcItem *p;
+  SelectDest dest;
+  Select *pSelect = 0;
+
+  if( pParse->db->init.busy 
+   || pParse->pNewTrigger
+   || pParse->bHasWith
+   || multiValueIsConstant(pRow)==0
+   || pLeft->pPrior
+  ){
+    int f = SF_Values | SF_MultiValue;
+    if( pLeft->pPrior || pLeft->pSrc->nSrc ){
+      sqlite3MultiValuesEnd(pParse, pLeft);
+      f = SF_Values;
+    }
+    /* This VALUES clause is part of a VIEW or some other schema item. In
+    ** this case the co-routine cannot be coded immediately.  */
+    pSelect = sqlite3SelectNew(pParse,pRow,0,0,0,0,0,f,0);
+    pLeft->selFlags &= ~SF_MultiValue;
+    if( pSelect ){
+      pSelect->op = TK_ALL;
+      pSelect->pPrior = pLeft;
+      pLeft = pSelect;
+    }
+  }else{
+
+    if( pLeft->pSrc->nSrc==0 ){
+      /* Co-routine has not yet been started. */
+      Vdbe *v = sqlite3GetVdbe(pParse);
+      Select *pRet;
+  
+      if( v==0 ) return pLeft;
+      pRet = sqlite3SelectNew(pParse, 0, 0, 0, 0, 0, 0, 0, 0);
+      if( pRet==0 ) return pLeft;
+      p = &pRet->pSrc->a[0];
+      pRet->pSrc->nSrc = 1;
+  
+      p->pSelect = pLeft;
+      p->fg.viaCoroutine = 1;
+      p->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1;
+      p->regReturn = ++pParse->nMem;
+  
+      sqlite3VdbeAddOp3(v,OP_InitCoroutine,p->regReturn,0,p->addrFillSub);
+      sqlite3SelectDestInit(&dest, SRT_Coroutine, p->regReturn);
+      sqlite3Select(pParse, pLeft, &dest);
+      p->regResult = dest.iSdst;
+      assert( pParse->nErr || dest.iSdst>0 );
+  
+      pLeft = pRet;
+    }else{
+      p = &pLeft->pSrc->a[0];
+    }
+  
+    if( pParse->nErr==0 ){
+      pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, SF_Values, 0);
+      if( pSelect ){
+        if( p->pSelect->pEList->nExpr!=pSelect->pEList->nExpr ){
+          sqlite3SelectWrongNumTermsError(pParse, pSelect);
+        }else{
+          sqlite3SelectPrep(pParse, pSelect, 0);
+#ifndef SQLITE_OMIT_WINDOWFUNC
+          if( pSelect->pWin ){
+            sqlite3SelectDestInit(&dest, SRT_Coroutine, p->regReturn);
+            dest.iSdst = p->regResult;
+            dest.nSdst = pRow->nExpr;
+            dest.iSDParm = p->regReturn;
+            sqlite3Select(pParse, pSelect, &dest);
+          }else
+#endif
+          {
+            sqlite3ExprCodeExprList(pParse, pSelect->pEList,p->regResult,0,0);
+            sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, p->regReturn);
+          }
+        }
+        sqlite3SelectDelete(pParse->db, pSelect);
+      }
+    }
+  }
+
+  return pLeft;
+}
 
 /* Forward declaration */
 static int xferOptimization(
@@ -914,24 +1012,31 @@ void sqlite3Insert(
     /* Data is coming from a SELECT or from a multi-row VALUES clause.
     ** Generate a co-routine to run the SELECT. */
     int regYield;       /* Register holding co-routine entry-point */
-    int addrTop;        /* Top of the co-routine */
     int rc;             /* Result code */
 
-    regYield = ++pParse->nMem;
-    addrTop = sqlite3VdbeCurrentAddr(v) + 1;
-    sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
-    sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
-    dest.iSdst = bIdListInOrder ? regData : 0;
-    dest.nSdst = pTab->nCol;
-    rc = sqlite3Select(pParse, pSelect, &dest);
-    regFromSelect = dest.iSdst;
-    assert( db->pParse==pParse );
-    if( rc || pParse->nErr ) goto insert_cleanup;
-    assert( db->mallocFailed==0 );
-    sqlite3VdbeEndCoroutine(v, regYield);
-    sqlite3VdbeJumpHere(v, addrTop - 1);                       /* label B: */
-    assert( pSelect->pEList );
-    nColumn = pSelect->pEList->nExpr;
+    if( pSelect->pSrc->nSrc==1 && pSelect->pSrc->a[0].fg.viaCoroutine ){
+      SrcItem *pItem = &pSelect->pSrc->a[0];
+      dest.iSDParm = regYield = pItem->regReturn;
+      regFromSelect = pItem->regResult;
+      nColumn = pItem->pSelect->pEList->nExpr;
+    }else{
+      int addrTop;        /* Top of the co-routine */
+      regYield = ++pParse->nMem;
+      addrTop = sqlite3VdbeCurrentAddr(v) + 1;
+      sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
+      sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
+      dest.iSdst = bIdListInOrder ? regData : 0;
+      dest.nSdst = pTab->nCol;
+      rc = sqlite3Select(pParse, pSelect, &dest);
+      regFromSelect = dest.iSdst;
+      assert( db->pParse==pParse );
+      if( rc || pParse->nErr ) goto insert_cleanup;
+      assert( db->mallocFailed==0 );
+      sqlite3VdbeEndCoroutine(v, regYield);
+      sqlite3VdbeJumpHere(v, addrTop - 1);                       /* label B: */
+      assert( pSelect->pEList );
+      nColumn = pSelect->pEList->nExpr;
+    }
 
     /* Set useTempTable to TRUE if the result of the SELECT statement
     ** should be written into a temporary table (template 4).  Set to
diff --git a/src/parse.y b/src/parse.y
index 37c9fa8bc..515d512f2 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -562,9 +562,13 @@ cmd ::= select(X).  {
 }
 
 %ifndef SQLITE_OMIT_CTE
-select(A) ::= WITH wqlist(W) selectnowith(X). {A = attachWithToSelect(pParse,X,W);}
-select(A) ::= WITH RECURSIVE wqlist(W) selectnowith(X).
+select(A) ::= withkw wqlist(W) selectnowith(X). {A = attachWithToSelect(pParse,X,W);}
+select(A) ::= withkw RECURSIVE wqlist(W) selectnowith(X).
                                               {A = attachWithToSelect(pParse,X,W);}
+
+withkw ::= WITH. {
+  pParse->bHasWith = 1;
+}
 %endif /* SQLITE_OMIT_CTE */
 select(A) ::= selectnowith(A). {
   Select *p = A;
@@ -622,7 +626,9 @@ oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y)
 %endif
 
 
-oneselect(A) ::= values(A).
+oneselect(A) ::= values(A). {
+  sqlite3MultiValuesEnd(pParse, A);
+}
 
 %type values {Select*}
 %destructor values {sqlite3SelectDelete(pParse->db, $$);}
@@ -630,16 +636,7 @@ values(A) ::= VALUES LP nexprlist(X) RP. {
   A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0);
 }
 values(A) ::= values(A) COMMA LP nexprlist(Y) RP. {
-  Select *pRight, *pLeft = A;
-  pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0);
-  if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue;
-  if( pRight ){
-    pRight->op = TK_ALL;
-    pRight->pPrior = pLeft;
-    A = pRight;
-  }else{
-    A = pLeft;
-  }
+  A = sqlite3MultiValues(pParse, A, Y);
 }
 
 // The "distinct" nonterminal is true (1) if the DISTINCT keyword is
@@ -1754,8 +1751,8 @@ anylist ::= anylist ANY.
 
 with ::= .
 %ifndef SQLITE_OMIT_CTE
-with ::= WITH wqlist(W).              { sqlite3WithPush(pParse, W, 1); }
-with ::= WITH RECURSIVE wqlist(W).    { sqlite3WithPush(pParse, W, 1); }
+with ::= withkw wqlist(W).              { sqlite3WithPush(pParse, W, 1); }
+with ::= withkw RECURSIVE wqlist(W).    { sqlite3WithPush(pParse, W, 1); }
 
 %type wqas {u8}
 wqas(A)   ::= AS.                  {A = M10d_Any;}
diff --git a/src/select.c b/src/select.c
index 81e802d6e..4158dd2fd 100644
--- a/src/select.c
+++ b/src/select.c
@@ -7636,7 +7636,7 @@ int sqlite3Select(
     /* Generate code for all sub-queries in the FROM clause
     */
     pSub = pItem->pSelect;
-    if( pSub==0 ) continue;
+    if( pSub==0 || pItem->addrFillSub!=0 ) continue;
 
     /* The code for a subquery should only be generated once. */
     assert( pItem->addrFillSub==0 );
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index e11b5b3a4..391339271 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3812,6 +3812,7 @@ struct Parse {
   u8 disableLookaside; /* Number of times lookaside has been disabled */
   u8 prepFlags;        /* SQLITE_PREPARE_* flags */
   u8 withinRJSubrtn;   /* Nesting level for RIGHT JOIN body subroutines */
+  u8 bHasWith;         /* True if statement contains WITH */
 #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
   u8 earlyCleanup;     /* OOM inside sqlite3ParserAddCleanup() */
 #endif
@@ -4486,6 +4487,9 @@ struct Window {
                           ** due to the SQLITE_SUBTYPE flag */
 };
 
+Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow);
+void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal);
+
 #ifndef SQLITE_OMIT_WINDOWFUNC
 void sqlite3WindowDelete(sqlite3*, Window*);
 void sqlite3WindowUnlinkFromSelect(Window*);
diff --git a/src/where.c b/src/where.c
index 9850d22cc..9abc8e30f 100644
--- a/src/where.c
+++ b/src/where.c
@@ -6896,6 +6896,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     */
     if( pTabItem->fg.viaCoroutine ){
       testcase( pParse->db->mallocFailed );
+      assert( pTabItem->regResult>=0 );
       translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
                             pTabItem->regResult, 0);
       continue;