Allow underscores to occur in hex literals.

FossilOrigin-Name: 81a56229460cc5b6acfd3c3729fcf89ea3cccb546ca2b4f4035b140c60911e18

2 files changed, 50 insertions, 36 deletions

diff --git a/src/tokenize.c b/src/tokenize.c
index 7a30512ec..f01548d4c 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -437,30 +437,11 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
       *tokenType = TK_INTEGER;
 #ifndef SQLITE_OMIT_HEX_INTEGER
       if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
-        for(i=3; sqlite3Isxdigit(z[i]); i++){}
-        return i;
-      }
-#endif
-      for(i=0; 1; i++){
-        if( sqlite3Isdigit(z[i])==0 ){
-          if( z[i]==SQLITE_DIGIT_SEPARATOR
-           && sqlite3Isdigit(z[i-1])
-           && sqlite3Isdigit(z[i+1])
-          ){
-            *tokenType = TK_QNUMBER;
-          }else{
-            break;
-          }
-        }
-      }
-#ifndef SQLITE_OMIT_FLOATING_POINT
-      if( z[i]=='.' ){
-        if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT;
-        for(i++; 1; i++){
-          if( sqlite3Isdigit(z[i])==0 ){
+        for(i=3; 1; i++){
+          if( sqlite3Isxdigit(z[i])==0 ){
             if( z[i]==SQLITE_DIGIT_SEPARATOR
-             && sqlite3Isdigit(z[i-1])
-             && sqlite3Isdigit(z[i+1])
+             && sqlite3Isxdigit(z[i-1])
+             && sqlite3Isxdigit(z[i+1])
             ){
               *tokenType = TK_QNUMBER;
             }else{
@@ -468,14 +449,10 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
             }
           }
         }
-      }
-      if( (z[i]=='e' || z[i]=='E') &&
-           ( sqlite3Isdigit(z[i+1]) 
-            || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
-           )
-      ){
-        if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT;
-        for(i+=2; 1; i++){
+      }else
+#endif
+        {
+        for(i=0; 1; i++){
           if( sqlite3Isdigit(z[i])==0 ){
             if( z[i]==SQLITE_DIGIT_SEPARATOR
              && sqlite3Isdigit(z[i-1])
@@ -487,8 +464,43 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
             }
           }
         }
-      }
+#ifndef SQLITE_OMIT_FLOATING_POINT
+        if( z[i]=='.' ){
+          if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT;
+          for(i++; 1; i++){
+            if( sqlite3Isdigit(z[i])==0 ){
+              if( z[i]==SQLITE_DIGIT_SEPARATOR
+               && sqlite3Isdigit(z[i-1])
+               && sqlite3Isdigit(z[i+1])
+              ){
+                *tokenType = TK_QNUMBER;
+              }else{
+                break;
+              }
+            }
+          }
+        }
+        if( (z[i]=='e' || z[i]=='E') &&
+             ( sqlite3Isdigit(z[i+1]) 
+              || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
+             )
+        ){
+          if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT;
+          for(i+=2; 1; i++){
+            if( sqlite3Isdigit(z[i])==0 ){
+              if( z[i]==SQLITE_DIGIT_SEPARATOR
+               && sqlite3Isdigit(z[i-1])
+               && sqlite3Isdigit(z[i+1])
+              ){
+                *tokenType = TK_QNUMBER;
+              }else{
+                break;
+              }
+            }
+          }
+        }
 #endif
+      }
       while( IdChar(z[i]) ){
         *tokenType = TK_ILLEGAL;
         i++;
diff --git a/src/util.c b/src/util.c
index 24eff14e9..9a2ea9e90 100644
--- a/src/util.c
+++ b/src/util.c
@@ -312,10 +312,9 @@ void sqlite3DequoteExpr(Expr *p){
 }
 
 /*
-** Expression p is a QINTEGER or QFLOAT (quoted integer or float). Dequote
-** the value in p->u.zToken and set the type to INTEGER or FLOAT. "Quoted"
-** integers or floats are those that contain '_' characters that must
-** be removed before further processing.
+** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken
+** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those
+** that contain '_' characters that must be removed before further processing.
 */
 void sqlite3DequoteNumber(Expr *p){
   if( p ){
@@ -329,6 +328,9 @@ void sqlite3DequoteNumber(Expr *p){
         if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT;
       }
     }while( *pIn++ );
+    if( p->u.zToken[0]=='0' && p->u.zToken[1]=='x' ){
+      p->op = TK_INTEGER;
+    }
   }
 }