Record within the Token structure itself whether or not the token has

been dequoted.  This steals one bit from the length of a token and
thus limits the size of tokens to 1GiB. (CVS 6589)

FossilOrigin-Name: 12bcb03d9b9e1a31c1a3c67cbb4263cc0af2f3d0

11 files changed, 56 insertions, 91 deletions

diff --git a/src/build.c b/src/build.c
index f6e325aa8..01222593a 100644
--- a/src/build.c
+++ b/src/build.c
@@ -22,7 +22,7 @@
 **     COMMIT
 **     ROLLBACK
 **
-** $Id: build.c,v 1.532 2009/04/28 13:01:09 drh Exp $
+** $Id: build.c,v 1.533 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -579,10 +579,13 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){
 
 /*
 ** Given a token, return a string that consists of the text of that
-** token with any quotations removed.  Space to hold the returned string
+** token.  Space to hold the returned string
 ** is obtained from sqliteMalloc() and must be freed by the calling
 ** function.
 **
+** Any quotation marks (ex:  "name", 'name', [name], or `name`) that
+** surround the body of the token are removed.
+**
 ** Tokens are often just pointers into the original SQL text and so
 ** are not \000 terminated and are not persistent.  The returned string
 ** is \000 terminated and is persistent.
@@ -591,7 +594,7 @@ char *sqlite3NameFromToken(sqlite3 *db, Token *pName){
   char *zName;
   if( pName ){
     zName = sqlite3DbStrNDup(db, (char*)pName->z, pName->n);
-    sqlite3Dequote(zName);
+    if( pName->quoted ) sqlite3Dequote(zName);
   }else{
     zName = 0;
   }
diff --git a/src/delete.c b/src/delete.c
index b0a05a470..cf0a36ef0 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -12,7 +12,7 @@
 ** This file contains C code routines that are called by the parser
 ** in order to generate code for DELETE FROM statements.
 **
-** $Id: delete.c,v 1.200 2009/04/30 00:11:10 drh Exp $
+** $Id: delete.c,v 1.201 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -87,6 +87,7 @@ void sqlite3MaterializeView(
     pWhere = sqlite3ExprDup(db, pWhere, 0);
     viewName.z = (u8*)pView->zName;
     viewName.n = (unsigned int)sqlite3Strlen30((const char*)viewName.z);
+    viewName.quoted = 0;
     pFrom = sqlite3SrcListAppendFromTerm(pParse, 0, 0, 0, &viewName, pDup, 0,0);
     pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0);
   }
diff --git a/src/expr.c b/src/expr.c
index ff5c4bd51..fa15cad8d 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -12,7 +12,7 @@
 ** This file contains routines used for analyzing expressions and
 ** for generating VDBE code that evaluates expressions in SQLite.
 **
-** $Id: expr.c,v 1.430 2009/04/28 12:08:15 danielk1977 Exp $
+** $Id: expr.c,v 1.431 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -407,21 +407,18 @@ Expr *sqlite3Expr(
     int c;
     assert( pToken->dyn==0 );
     pNew->span = *pToken;
- 
-    /* The pToken->z value is read-only.  But the new expression
-    ** node created here might be passed to sqlite3DequoteExpr() which
-    ** will attempt to modify pNew->token.z.  Hence, if the token
-    ** is quoted, make a copy now so that DequoteExpr() will change
-    ** the copy rather than the original text.
-    */
     if( pToken->n>=2 
          && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){
       sqlite3TokenCopy(db, &pNew->token, pToken);
+      if( pNew->token.z ){
+        pNew->token.n = sqlite3Dequote((char*)pNew->token.z);
+        assert( pNew->token.n==sqlite3Strlen30((char*)pNew->token.z) );
+      }
+      if( c=='"' ) pNew->flags |= EP_DblQuoted;
     }else{
       pNew->token = *pToken;
-      pNew->flags |= EP_Dequoted;
-      VVA_ONLY( pNew->vvaFlags |= EVVA_ReadOnlyToken; )
     }
+    pNew->token.quoted = 0;
   }else if( pLeft ){
     if( pRight ){
       if( pRight->span.dyn==0 && pLeft->span.dyn==0 ){
@@ -659,18 +656,6 @@ void sqlite3ExprDelete(sqlite3 *db, Expr *p){
 }
 
 /*
-** The Expr.token field might be a string literal that is quoted.
-** If so, remove the quotation marks.
-*/
-void sqlite3DequoteExpr(Expr *p){
-  if( !ExprHasAnyProperty(p, EP_Dequoted) ){
-    ExprSetProperty(p, EP_Dequoted);
-    assert( (p->vvaFlags & EVVA_ReadOnlyToken)==0 );
-    sqlite3Dequote((char*)p->token.z);
-  }
-}
-
-/*
 ** Return the number of bytes allocated for the expression structure 
 ** passed as the first argument. This is always one of EXPR_FULLSIZE,
 ** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE.
@@ -1595,7 +1580,7 @@ void sqlite3CodeSubselect(
       ** value of this select in a memory cell and record the number
       ** of the memory cell in iColumn.
       */
-      static const Token one = { (u8*)"1", 0, 1 };
+      static const Token one = { (u8*)"1", 0, 0, 1 };
       Select *pSel;
       SelectDest dest;
 
@@ -2082,8 +2067,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       break;
     }
     case TK_STRING: {
-      sqlite3DequoteExpr(pExpr);
-      sqlite3VdbeAddOp4(v,OP_String8, 0, target, 0,
+      sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0,
                         (char*)pExpr->token.z, pExpr->token.n);
       break;
     }
@@ -2591,7 +2575,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
          assert( pExpr->affinity==OE_Rollback ||
                  pExpr->affinity == OE_Abort ||
                  pExpr->affinity == OE_Fail );
-         sqlite3DequoteExpr(pExpr);
          sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, pExpr->affinity, 0,
                         (char*)pExpr->token.z, pExpr->token.n);
       } else {
diff --git a/src/parse.y b/src/parse.y
index 702481437..26017bdcb 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -14,7 +14,7 @@
 ** the parser.  Lemon will also generate a header file containing
 ** numeric codes for all of the tokens.
 **
-** @(#) $Id: parse.y,v 1.274 2009/04/06 14:16:43 drh Exp $
+** @(#) $Id: parse.y,v 1.275 2009/05/01 21:13:37 drh Exp $
 */
 
 // All token codes are small integers with #defines that begin with "TK_"
@@ -166,6 +166,8 @@ columnlist ::= column.
 column(A) ::= columnid(X) type carglist. {
   A.z = X.z;
   A.n = (int)(pParse->sLastToken.z-X.z) + pParse->sLastToken.n;
+  A.quoted = 0;
+  A.dyn = 0;
 }
 columnid(A) ::= nm(X). {
   sqlite3AddColumn(pParse,&X);
diff --git a/src/resolve.c b/src/resolve.c
index 1f2641df3..ac6ada05e 100644
--- a/src/resolve.c
+++ b/src/resolve.c
@@ -14,7 +14,7 @@
 ** resolve all identifiers by associating them with a particular
 ** table and column.
 **
-** $Id: resolve.c,v 1.20 2009/03/05 04:23:47 shane Exp $
+** $Id: resolve.c,v 1.21 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 #include <stdlib.h>
@@ -319,7 +319,7 @@ static int lookupName(
   ** Because no reference was made to outer contexts, the pNC->nRef
   ** fields are not changed in any context.
   */
-  if( cnt==0 && zTab==0 && pColumnToken->z[0]=='"' ){
+  if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){
     sqlite3DbFree(db, zCol);
     pExpr->op = TK_STRING;
     pExpr->pTab = 0;
diff --git a/src/select.c b/src/select.c
index 3c806b187..54600e53d 100644
--- a/src/select.c
+++ b/src/select.c
@@ -12,7 +12,7 @@
 ** This file contains C code routines that are called by the parser
 ** to handle SELECT statements in SQLite.
 **
-** $Id: select.c,v 1.510 2009/04/24 15:46:22 drh Exp $
+** $Id: select.c,v 1.511 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -193,42 +193,7 @@ static void setToken(Token *p, const char *z){
   p->z = (u8*)z;
   p->n = z ? sqlite3Strlen30(z) : 0;
   p->dyn = 0;
-}
-
-/*
-** Set the token to the double-quoted and escaped version of the string pointed
-** to by z. For example;
-**
-**    {a"bc}  ->  {"a""bc"}
-*/
-static void setQuotedToken(Parse *pParse, Token *p, const char *z){
-
-  /* Check if the string appears to be quoted using "..." or `...`
-  ** or [...] or '...' or if the string contains any " characters.  
-  ** If it does, then record a version of the string with the special
-  ** characters escaped.
-  */
-  const char *z2 = z;
-  if( *z2!='[' && *z2!='`' && *z2!='\'' ){
-    while( *z2 ){
-      if( *z2=='"' ) break;
-      z2++;
-    }
-  }
-
-  if( *z2 ){
-    /* String contains " characters - copy and quote the string. */
-    p->z = (u8 *)sqlite3MPrintf(pParse->db, "\"%w\"", z);
-    if( p->z ){
-      p->n = sqlite3Strlen30((char *)p->z);
-      p->dyn = 1;
-    }
-  }else{
-    /* String contains no " characters - copy the pointer. */
-    p->z = (u8*)z;
-    p->n = (int)(z2 - z);
-    p->dyn = 0;
-  }
+  p->quoted = 0;
 }
 
 /*
@@ -3228,12 +3193,12 @@ static int selectExpander(Walker *pWalker, Select *p){
             }
             pRight = sqlite3PExpr(pParse, TK_ID, 0, 0, 0);
             if( pRight==0 ) break;
-            setQuotedToken(pParse, &pRight->token, zName);
+            setToken(&pRight->token, zName);
             if( longNames || pTabList->nSrc>1 ){
               Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, 0);
               pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0);
               if( pExpr==0 ) break;
-              setQuotedToken(pParse, &pLeft->token, zTabName);
+              setToken(&pLeft->token, zTabName);
               setToken(&pExpr->span, 
                   sqlite3MPrintf(db, "%s.%s", zTabName, zName));
               pExpr->span.dyn = 1;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index afdb4b3c9..488a0686e 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.864 2009/04/30 12:25:10 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.865 2009/05/01 21:13:37 drh Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -1354,8 +1354,9 @@ struct Index {
 */
 struct Token {
   const unsigned char *z; /* Text of the token.  Not NULL-terminated! */
-  unsigned dyn  : 1;      /* True for malloced memory, false for static */
-  unsigned n    : 31;     /* Number of characters in this token */
+  unsigned dyn    : 1;    /* True for malloced memory, false for static */
+  unsigned quoted : 1;    /* True if token still has its quotes */
+  unsigned n      : 30;   /* Number of characters in this token */
 };
 
 /*
@@ -1517,7 +1518,7 @@ struct Expr {
 #define EP_Error      0x0008  /* Expression contains one or more errors */
 #define EP_Distinct   0x0010  /* Aggregate function with DISTINCT keyword */
 #define EP_VarSelect  0x0020  /* pSelect is correlated, not constant */
-#define EP_Dequoted   0x0040  /* True if the string has been dequoted */
+#define EP_DblQuoted  0x0040  /* token.z was originally in "..." */
 #define EP_InfixFunc  0x0080  /* True for an infix function: LIKE, GLOB, etc */
 #define EP_ExpCollate 0x0100  /* Collating sequence specified explicitly */
 #define EP_AnyAff     0x0200  /* Can take a cached column of any affinity */
@@ -2360,8 +2361,7 @@ char *sqlite3MAppendf(sqlite3*,char*,const char*,...);
 void sqlite3SetString(char **, sqlite3*, const char*, ...);
 void sqlite3ErrorMsg(Parse*, const char*, ...);
 void sqlite3ErrorClear(Parse*);
-void sqlite3Dequote(char*);
-void sqlite3DequoteExpr(Expr*);
+int sqlite3Dequote(char*);
 int sqlite3KeywordCode(const unsigned char*, int);
 int sqlite3RunParser(Parse*, const char*, char **);
 void sqlite3FinishCoding(Parse*);
diff --git a/src/tokenize.c b/src/tokenize.c
index 4677811d2..a933de1b9 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -15,7 +15,7 @@
 ** individual tokens and sends those tokens one-by-one over to the
 ** parser for analysis.
 **
-** $Id: tokenize.c,v 1.155 2009/03/31 03:41:57 shane Exp $
+** $Id: tokenize.c,v 1.156 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 #include <stdlib.h>
@@ -413,10 +413,12 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   assert( pParse->apVarExpr==0 );
   enableLookaside = db->lookaside.bEnabled;
   if( db->lookaside.pStart ) db->lookaside.bEnabled = 1;
+  pParse->sLastToken.quoted = 1;
   while( !db->mallocFailed && zSql[i]!=0 ){
     assert( i>=0 );
     pParse->sLastToken.z = (u8*)&zSql[i];
     assert( pParse->sLastToken.dyn==0 );
+    assert( pParse->sLastToken.quoted );
     pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType);
     i += pParse->sLastToken.n;
     if( i>mxSqlLen ){
diff --git a/src/util.c b/src/util.c
index 6b18cf270..b2ab11101 100644
--- a/src/util.c
+++ b/src/util.c
@@ -14,7 +14,7 @@
 ** This file contains functions for allocating memory, comparing
 ** strings, and stuff like that.
 **
-** $Id: util.c,v 1.251 2009/04/17 15:18:48 drh Exp $
+** $Id: util.c,v 1.252 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 #include <stdarg.h>
@@ -203,21 +203,28 @@ void sqlite3ErrorClear(Parse *pParse){
 ** input does not begin with a quote character, then this routine
 ** is a no-op.
 **
+** The input string must be zero-terminated.  A new zero-terminator
+** is added to the dequoted string.
+**
+** The return value is -1 if no dequoting occurs or the length of the
+** dequoted string, exclusive of the zero terminator, if dequoting does
+** occur.
+**
 ** 2002-Feb-14: This routine is extended to remove MS-Access style
 ** brackets from around identifers.  For example:  "[a-b-c]" becomes
 ** "a-b-c".
 */
-void sqlite3Dequote(char *z){
+int sqlite3Dequote(char *z){
   char quote;
   int i, j;
-  if( z==0 ) return;
+  if( z==0 ) return -1;
   quote = z[0];
   switch( quote ){
     case '\'':  break;
     case '"':   break;
     case '`':   break;                /* For MySQL compatibility */
     case '[':   quote = ']';  break;  /* For MS SqlServer compatibility */
-    default:    return;
+    default:    return -1;
   }
   for(i=1, j=0; z[i]; i++){
     if( z[i]==quote ){
@@ -225,13 +232,14 @@ void sqlite3Dequote(char *z){
         z[j++] = quote;
         i++;
       }else{
-        z[j++] = 0;
         break;
       }
     }else{
       z[j++] = z[i];
     }
   }
+  z[j] = 0;
+  return j;
 }
 
 /* Convenient short-hand */
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 6bc0e1ff8..9ad47f562 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -15,7 +15,7 @@
 ** only within the VDBE.  Interface routines refer to a Mem using the
 ** name sqlite_value
 **
-** $Id: vdbemem.c,v 1.142 2009/04/22 02:15:49 drh Exp $
+** $Id: vdbemem.c,v 1.143 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
@@ -977,7 +977,6 @@ int sqlite3ValueFromExpr(
     zVal = sqlite3DbStrNDup(db, (char*)pExpr->token.z, pExpr->token.n);
     pVal = sqlite3ValueNew(db);
     if( !zVal || !pVal ) goto no_mem;
-    sqlite3Dequote(zVal);
     sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC);
     if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){
       sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, enc);
diff --git a/src/where.c b/src/where.c
index 3f850d6d2..436c24684 100644
--- a/src/where.c
+++ b/src/where.c
@@ -16,7 +16,7 @@
 ** so is applicable.  Because this module is responsible for selecting
 ** indices, you might also think of this module as the "query optimizer".
 **
-** $Id: where.c,v 1.391 2009/04/29 11:50:54 danielk1977 Exp $
+** $Id: where.c,v 1.392 2009/05/01 21:13:37 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -625,6 +625,7 @@ static int isLikeOrGlob(
   Expr *pRight, *pLeft;      /* Right and left size of LIKE operator */
   ExprList *pList;           /* List of operands to the LIKE operator */
   int c;                     /* One character in z[] */
+  int n;                     /* Length of string z[] */
   int cnt;                   /* Number of non-wildcard prefix characters */
   char wc[3];                /* Wildcard characters */
   CollSeq *pColl;            /* Collating sequence for LHS */
@@ -655,11 +656,13 @@ static int isLikeOrGlob(
       (pColl->type!=SQLITE_COLL_NOCASE || !*pnoCase) ){
     return 0;
   }
-  sqlite3DequoteExpr(pRight);
-  z = (char *)pRight->token.z;
+  z = (const char*)pRight->token.z;
   cnt = 0;
   if( z ){
-    while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; }
+    n = pRight->token.n;
+    while( cnt<n && (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){
+      cnt++;
+    }
   }
   if( cnt==0 || 255==(u8)z[cnt-1] ){
     return 0;
@@ -1160,7 +1163,6 @@ static void exprAnalyze(
     if( pStr1 ){
       sqlite3TokenCopy(db, &pStr1->token, &pRight->token);
       pStr1->token.n = nPattern;
-      pStr1->flags = EP_Dequoted;
     }
     pStr2 = sqlite3ExprDup(db, pStr1, 0);
     if( !db->mallocFailed ){