Enhances the parser so that it accepts arbitrary expressions for the arguments

of an index, though the code generator still rejects everything other than
simple column names.  The sqlite3RestrictColumnListSyntax() routine is removed
since that feature is now handled by the parser.

FossilOrigin-Name: bed42116addabcf3dfdc2e2d51ae183965704988

4 files changed, 100 insertions, 81 deletions

diff --git a/src/build.c b/src/build.c
index 815b17deb..8a7dda89c 100644
--- a/src/build.c
+++ b/src/build.c
@@ -1310,11 +1310,15 @@ void sqlite3AddPrimaryKey(
   }else{
     nTerm = pList->nExpr;
     for(i=0; i<nTerm; i++){
-      for(iCol=0; iCol<pTab->nCol; iCol++){
-        if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){
-          pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY;
-          zType = pTab->aCol[iCol].zType;
-          break;
+      Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[i].pExpr);
+      if( pCExpr && pCExpr->op==TK_ID ){
+        const char *zCName = pCExpr->u.zToken;
+        for(iCol=0; iCol<pTab->nCol; iCol++){
+          if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){
+            pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY;
+            zType = pTab->aCol[iCol].zType;
+            break;
+          }
         }
       }
     }
@@ -1696,10 +1700,12 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
   */
   if( pTab->iPKey>=0 ){
     ExprList *pList;
-    pList = sqlite3ExprListAppend(pParse, 0, 0);
+    Token ipkToken;
+    ipkToken.z = pTab->aCol[pTab->iPKey].zName;
+    ipkToken.n = sqlite3Strlen30(ipkToken.z);
+    pList = sqlite3ExprListAppend(pParse, 0, 
+                  sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0));
     if( pList==0 ) return;
-    pList->a[0].zName = sqlite3DbStrDup(pParse->db,
-                                        pTab->aCol[pTab->iPKey].zName);
     pList->a[0].sortOrder = pParse->iPkSortOrder;
     assert( pParse->pNewTable==pTab );
     pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0);
@@ -2076,7 +2082,6 @@ void sqlite3CreateView(
     return;
   }
   sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr);
-  sqlite3RestrictColumnListSyntax(pParse, pCNames);
   p = pParse->pNewTable;
   if( p==0 || pParse->nErr ) goto create_view_fail;
   sqlite3TwoPartName(pParse, pName1, pName2, &pName);
@@ -2607,8 +2612,6 @@ void sqlite3CreateForeignKey(
 
   assert( pTo!=0 );
   if( p==0 || IN_DECLARE_VTAB ) goto fk_end;
-  sqlite3RestrictColumnListSyntax(pParse, pFromCol);
-  sqlite3RestrictColumnListSyntax(pParse, pToCol);
   if( pFromCol==0 ){
     int iCol = p->nCol-1;
     if( NEVER(iCol<0) ) goto fk_end;
@@ -3043,12 +3046,16 @@ Index *sqlite3CreateIndex(
   ** So create a fake list to simulate this.
   */
   if( pList==0 ){
-    pList = sqlite3ExprListAppend(pParse, 0, 0);
+    Token prevCol;
+    prevCol.z = pTab->aCol[pTab->nCol-1].zName;
+    prevCol.n = sqlite3Strlen30(prevCol.z);
+    pList = sqlite3ExprListAppend(pParse, 0,
+              sqlite3ExprAlloc(db, TK_ID, &prevCol, 0));
     if( pList==0 ) goto exit_create_index;
-    pList->a[0].zName = sqlite3DbStrDup(pParse->db,
-                                        pTab->aCol[pTab->nCol-1].zName);
     assert( pList->nExpr==1 );
     sqlite3ExprListSetSortOrder(pList, sortOrder);
+  }else{
+    sqlite3ExprListCheckLength(pParse, pList, "index");
   }
 
   /* Figure out how many bytes of space are required to store explicitly
@@ -3056,8 +3063,7 @@ Index *sqlite3CreateIndex(
   */
   for(i=0; i<pList->nExpr; i++){
     Expr *pExpr = pList->a[i].pExpr;
-    if( pExpr ){
-      assert( pExpr->op==TK_COLLATE );
+    if( pExpr && pExpr->op==TK_COLLATE ){
       nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken));
     }
   }
@@ -3109,10 +3115,17 @@ Index *sqlite3CreateIndex(
   ** break backwards compatibility - it needs to be a warning.
   */
   for(i=0, pListItem=pList->a; i<pList->nExpr; i++, pListItem++){
-    const char *zColName = pListItem->zName;
+    const char *zColName;
+    Expr *pCExpr;
     int requestedSortOrder;
     char *zColl;                   /* Collation sequence name */
 
+    pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr);
+    if( pCExpr->op!=TK_ID ){
+      sqlite3ErrorMsg(pParse, "indexes on expressions not yet supported");
+      continue;
+    }
+    zColName = pCExpr->u.zToken;
     for(j=0, pTabCol=pTab->aCol; j<pTab->nCol; j++, pTabCol++){
       if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break;
     }
@@ -3124,9 +3137,8 @@ Index *sqlite3CreateIndex(
     }
     assert( j<=0x7fff );
     pIndex->aiColumn[i] = (i16)j;
-    if( pListItem->pExpr ){
+    if( pListItem->pExpr->op==TK_COLLATE ){
       int nColl;
-      assert( pListItem->pExpr->op==TK_COLLATE );
       zColl = pListItem->pExpr->u.zToken;
       nColl = sqlite3Strlen30(zColl) + 1;
       assert( nExtra>=nColl );
@@ -4293,32 +4305,6 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){
   return pKey;
 }
 
-/*
-** Generate a syntax error if the expression list provided contains
-** any COLLATE or ASC or DESC keywords.
-**
-** Some legacy versions of SQLite allowed constructs like:
-**
-**      CREATE TABLE x(..., FOREIGN KEY(x COLLATE binary DESC) REFERENCES...);
-**                                        ^^^^^^^^^^^^^^^^^^^
-**
-** The COLLATE and sort order terms were ignored.  To prevent compatibility
-** problems in case something like this appears in a legacy sqlite_master
-** table, only enforce the restriction on new SQL statements, not when
-** parsing the schema out of the sqlite_master table.
-*/
-void sqlite3RestrictColumnListSyntax(Parse *pParse, ExprList *p){
-  int i;
-  if( p==0 || pParse->db->init.busy ) return;
-  for(i=0; i<p->nExpr; i++){
-    if( p->a[i].pExpr!=0 || p->a[i].bDefinedSO ){
-      sqlite3ErrorMsg(pParse, "syntax error after column name \"%w\"",
-                      p->a[i].zName);
-      return;
-    }
-  }
-}
-
 #ifndef SQLITE_OMIT_CTE
 /* 
 ** This routine is invoked once per CTE by the parser while parsing a 
@@ -4335,8 +4321,6 @@ With *sqlite3WithAdd(
   With *pNew;
   char *zName;
 
-  sqlite3RestrictColumnListSyntax(pParse, pArglist);
-
   /* Check that the CTE name is unique within this WITH clause. If
   ** not, store an error in the Parse structure. */
   zName = sqlite3NameFromToken(pParse->db, pName);
diff --git a/src/expr.c b/src/expr.c
index 1c57ecc6f..1aebef6b1 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -1172,7 +1172,6 @@ void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){
     return;
   }
   p->a[p->nExpr-1].sortOrder = (u8)iSortOrder;
-  p->a[p->nExpr-1].bDefinedSO = 1;
 }
 
 /*
diff --git a/src/parse.y b/src/parse.y
index acd899449..e99feeefc 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -301,7 +301,7 @@ ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I).
                                  {sqlite3AddPrimaryKey(pParse,0,R,I,Z);}
 ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0);}
 ccons ::= CHECK LP expr(X) RP.   {sqlite3AddCheckConstraint(pParse,X.pExpr);}
-ccons ::= REFERENCES nm(T) idxlist_opt(TA) refargs(R).
+ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R).
                                  {sqlite3CreateForeignKey(pParse,0,&T,TA,R);}
 ccons ::= defer_subclause(D).    {sqlite3DeferForeignKey(pParse,D);}
 ccons ::= COLLATE ids(C).        {sqlite3AddCollateType(pParse, &C);}
@@ -345,14 +345,14 @@ conslist ::= tcons.
 tconscomma ::= COMMA.            {pParse->constraintName.n = 0;}
 tconscomma ::= .
 tcons ::= CONSTRAINT nm(X).      {pParse->constraintName = X;}
-tcons ::= PRIMARY KEY LP idxlist(X) autoinc(I) RP onconf(R).
+tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R).
                                  {sqlite3AddPrimaryKey(pParse,X,R,I,0);}
-tcons ::= UNIQUE LP idxlist(X) RP onconf(R).
+tcons ::= UNIQUE LP sortlist(X) RP onconf(R).
                                  {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0);}
 tcons ::= CHECK LP expr(E) RP onconf.
                                  {sqlite3AddCheckConstraint(pParse,E.pExpr);}
-tcons ::= FOREIGN KEY LP idxlist(FA) RP
-          REFERENCES nm(T) idxlist_opt(TA) refargs(R) defer_subclause_opt(D). {
+tcons ::= FOREIGN KEY LP eidlist(FA) RP
+          REFERENCES nm(T) eidlist_opt(TA) refargs(R) defer_subclause_opt(D). {
     sqlite3CreateForeignKey(pParse, FA, &T, TA, R);
     sqlite3DeferForeignKey(pParse, D);
 }
@@ -386,7 +386,7 @@ ifexists(A) ::= .            {A = 0;}
 ///////////////////// The CREATE VIEW statement /////////////////////////////
 //
 %ifndef SQLITE_OMIT_VIEW
-cmd ::= createkw(X) temp(T) VIEW ifnotexists(E) nm(Y) dbnm(Z) idxlist_opt(C)
+cmd ::= createkw(X) temp(T) VIEW ifnotexists(E) nm(Y) dbnm(Z) eidlist_opt(C)
           AS select(S). {
   sqlite3CreateView(pParse, &X, &Y, &Z, C, S, T, E);
 }
@@ -674,6 +674,11 @@ using_opt(U) ::= .                        {U = 0;}
 
 %type orderby_opt {ExprList*}
 %destructor orderby_opt {sqlite3ExprListDelete(pParse->db, $$);}
+
+// the sortlist non-terminal stores a list of expression where each
+// expression is optionally followed by ASC or DESC to indicate the
+// sort order.
+//
 %type sortlist {ExprList*}
 %destructor sortlist {sqlite3ExprListDelete(pParse->db, $$);}
 
@@ -1208,7 +1213,7 @@ nexprlist(A) ::= expr(Y).
 ///////////////////////////// The CREATE INDEX command ///////////////////////
 //
 cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D)
-        ON nm(Y) LP idxlist(Z) RP where_opt(W). {
+        ON nm(Y) LP sortlist(Z) RP where_opt(W). {
   sqlite3CreateIndex(pParse, &X, &D, 
                      sqlite3SrcListAppend(pParse->db,0,&Y,0), Z, U,
                       &S, W, SQLITE_SO_ASC, NE);
@@ -1218,31 +1223,64 @@ cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D)
 uniqueflag(A) ::= UNIQUE.  {A = OE_Abort;}
 uniqueflag(A) ::= .        {A = OE_None;}
 
-%type idxlist {ExprList*}
-%destructor idxlist {sqlite3ExprListDelete(pParse->db, $$);}
-%type idxlist_opt {ExprList*}
-%destructor idxlist_opt {sqlite3ExprListDelete(pParse->db, $$);}
-
-idxlist_opt(A) ::= .                         {A = 0;}
-idxlist_opt(A) ::= LP idxlist(X) RP.         {A = X;}
-idxlist(A) ::= idxlist(X) COMMA nm(Y) collate(C) sortorder(Z).  {
-  Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C, 1);
-  A = sqlite3ExprListAppend(pParse,X, p);
-  sqlite3ExprListSetName(pParse,A,&Y,1);
-  sqlite3ExprListCheckLength(pParse, A, "index");
-  sqlite3ExprListSetSortOrder(A,Z);
+
+// The eidlist non-terminal (Expression Id List) generates an ExprList
+// from a list of identifiers.  The identifier names are in ExprList.a[].zName.
+// This list is stored in an ExprList rather than an IdList so that it
+// can be easily sent to sqlite3ColumnsExprList().
+//
+// eidlist is grouped with CREATE INDEX because it used to be the non-terminal
+// used for the arguments to an index.  That is just an historical accident.
+//
+// IMPORTANT COMPATIBILITY NOTE:  Some prior versions of SQLite accepted
+// COLLATE clauses and ASC or DESC keywords on ID lists in inappropriate
+// places - places that might have been stored in the sqlite_master schema.
+// Those extra features were ignored.  But because they might be in some
+// (busted) old databases, we need to continue parsing them when loading
+// historical schemas.
+//
+%type eidlist {ExprList*}
+%destructor eidlist {sqlite3ExprListDelete(pParse->db, $$);}
+%type eidlist_opt {ExprList*}
+%destructor eidlist_opt {sqlite3ExprListDelete(pParse->db, $$);}
+
+%include {
+  /* Add a single new term to an ExprList that is used to store a
+  ** list of identifiers.  Report an error if the ID list contains
+  ** a COLLATE clause or an ASC or DESC keyword, except ignore the
+  ** error while parsing a legacy schema.
+  */
+  static ExprList *parserAddExprIdListTerm(
+    Parse *pParse,
+    ExprList *pPrior,
+    Token *pIdToken,
+    int hasCollate,
+    int sortOrder
+  ){
+    ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0);
+    if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED)
+        && pParse->db->init.busy==0
+    ){
+      sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"",
+                         pIdToken->n, pIdToken->z);
+    }
+    sqlite3ExprListSetName(pParse, p, pIdToken, 1);
+    return p;
+  }
+} // end %include
+
+eidlist_opt(A) ::= .                         {A = 0;}
+eidlist_opt(A) ::= LP eidlist(X) RP.         {A = X;}
+eidlist(A) ::= eidlist(X) COMMA nm(Y) collate(C) sortorder(Z).  {
+  A = parserAddExprIdListTerm(pParse, X, &Y, C, Z);
 }
-idxlist(A) ::= nm(Y) collate(C) sortorder(Z). {
-  Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C, 1);
-  A = sqlite3ExprListAppend(pParse,0, p);
-  sqlite3ExprListSetName(pParse, A, &Y, 1);
-  sqlite3ExprListCheckLength(pParse, A, "index");
-  sqlite3ExprListSetSortOrder(A,Z);
+eidlist(A) ::= nm(Y) collate(C) sortorder(Z). {
+  A = parserAddExprIdListTerm(pParse, 0, &Y, C, Z);
 }
 
-%type collate {Token}
-collate(C) ::= .                 {C.z = 0; C.n = 0;}
-collate(C) ::= COLLATE ids(X).   {C = X;}
+%type collate {int}
+collate(C) ::= .              {C = 0;}
+collate(C) ::= COLLATE ids.   {C = 1;}
 
 
 ///////////////////////////// The DROP INDEX command /////////////////////////
@@ -1490,10 +1528,10 @@ with(A) ::= . {A = 0;}
 with(A) ::= WITH wqlist(W).              { A = W; }
 with(A) ::= WITH RECURSIVE wqlist(W).    { A = W; }
 
-wqlist(A) ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. {
+wqlist(A) ::= nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
   A = sqlite3WithAdd(pParse, 0, &X, Y, Z);
 }
-wqlist(A) ::= wqlist(W) COMMA nm(X) idxlist_opt(Y) AS LP select(Z) RP. {
+wqlist(A) ::= wqlist(W) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
   A = sqlite3WithAdd(pParse, W, &X, Y, Z);
 }
 %endif  SQLITE_OMIT_CTE
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 5f1235053..4c17904ff 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2189,7 +2189,6 @@ struct ExprList {
     unsigned done :1;       /* A flag to indicate when processing is finished */
     unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */
     unsigned reusable :1;   /* Constant expression is reusable */
-    unsigned bDefinedSO :1; /* True if either DESC or ASC keywords present */
     union {
       struct {
         u16 iOrderByCol;      /* For ORDER BY, column number in result set */
@@ -3758,7 +3757,6 @@ const char *sqlite3JournalModename(int);
   int sqlite3Checkpoint(sqlite3*, int, int, int*, int*);
   int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int);
 #endif
-void sqlite3RestrictColumnListSyntax(Parse*,ExprList*);
 #ifndef SQLITE_OMIT_CTE
   With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*);
   void sqlite3WithDelete(sqlite3*,With*);