Work toward VT100-safe output from the CLI by default.

FossilOrigin-Name: 44c44620e8648a4265053f194e32b3a5c65d25b4f1fff61ef9b944e7cb0ed624

4 files changed, 136 insertions, 17 deletions

diff --git a/manifest b/manifest
index 82f49876a..bb780263e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C The\s%#Q\sconversion\snow\sadds\sunistr('...')\saround\sthe\sconverted\sstring\sif\nescape\scharacters\swere\sinserted.\s\s%#w\snow\sworks\sjust\slike\s%w\sas\sescape\nsequences\sinside\sof\sidentifiers\sare\snot\srecognized.
-D 2025-02-23T11:48:07.678
+C Work\stoward\sVT100-safe\soutput\sfrom\sthe\sCLI\sby\sdefault.
+D 2025-02-23T20:20:56.015
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
@@ -782,7 +782,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 626c24b258b111f75c22107aa5614ad89810df3026f5ca071116d3fe75925c75
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c a076f7db3a0fcbd9f710d7746cfc07e0b3baadee45eb3136bedc29c598ef8f1c
-F src/shell.c.in bf997e43faaa1ef0ff78d4d7b9be6a9430cf1edda9a47a14e7fef646fcb459af
+F src/shell.c.in b1ee6353204a6b543a9e07d44339473db5d3b71c63ca05bbb452391948b66682
 F src/sqlite.h.in 8d4486fb28a90de818ac1e8c6206ea458e7de6bd8e0dfa3d554494f155be8c01
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
@@ -1638,7 +1638,7 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21
 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707
 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939
 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
-F test/shell1.test 5d84e415adf7cc4edd5913c4f23c761104ff135b9c190fcf7b430a4cbca6cb65
+F test/shell1.test 069ad46cc5e576207e684168777b9d093ad419f4573ea96c47cba9d97b344ba9
 F test/shell2.test 01a01f76ed98088ce598794fbf5b359e148271541a8ddbf79d21cc353cc67a24
 F test/shell3.test db1953a8e59d08e9240b7cc5948878e184f7eb2623591587f8fd1f1a5bd536d8
 F test/shell4.test 522fdc628c55eff697b061504fb0a9e4e6dfc5d9087a633ab0f3dd11bcc4f807
@@ -2210,8 +2210,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ffbfcc2bbb57f02aa5ee813e7a25a2a014e3353a10f6bccb609075a5b63545d7
-R b7621246db8a84c2bec12427fadf6f60
+P 997391d42079783e294836f714ccd9526ecc442c8dbf8212d72cd17c67e7158a
+R fb09ed0e2944dd462b4d95bf157b726f
 U drh
-Z bdb58a5a11dd9515f24e08ce18cb765e
+Z 547baf0b0b8287880898ad372e983fdc
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 24c7fa98d..2b72ba576 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-997391d42079783e294836f714ccd9526ecc442c8dbf8212d72cd17c67e7158a
+44c44620e8648a4265053f194e32b3a5c65d25b4f1fff61ef9b944e7cb0ed624
diff --git a/src/shell.c.in b/src/shell.c.in
index f12d360b9..a8ea0dd2a 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -1446,6 +1446,7 @@ struct ShellState {
   u8 bSafeModePersist;   /* The long-term value of bSafeMode */
   u8 eRestoreState;      /* See comments above doAutoDetectRestore() */
   u8 crlfMode;           /* Do NL-to-CRLF translations when enabled (maybe) */
+  u8 eEscMode;           /* Escape mode for text output */
   ColModeOpts cmOpts;    /* Option values affecting columnar mode output */
   unsigned statsOn;      /* True to display memory stats before each finalize */
   unsigned mEqpLines;    /* Mask of vertical lines in the EQP output graph */
@@ -1546,6 +1547,11 @@ static ShellState shellState;
                                    ** top-level SQL statement */
 #define SHELL_PROGRESS_ONCE  0x04  /* Cancel the --limit after firing once */
 
+/* Allowed values for ShellState.eEscMode
+*/
+#define SHELL_ESC_GRAPHIC      0      /* Substitute U+2400 graphics */
+#define SHELL_ESC_OFF          1      /* Send characters verbatim */
+
 /*
 ** These are the allowed shellFlgs values
 */
@@ -2155,6 +2161,84 @@ static void output_json_string(FILE *out, const char *z, i64 n){
 }
 
 /*
+** Escape the input string if it is needed and in accordance with
+** eEscMode.
+**
+** Escaping is needed if the string contains any control characters
+** other than \t, \n, and \r\n
+**
+** If no escaping is needed (the common case) then set *ppFree to NULL
+** and return the original string.  If escapingn is needed, write the
+** escaped string into memory obtained from sqlite3_malloc64() or the
+** equivalent, and return the new string and set *ppFree to the new string
+** as well.
+**
+** The caller is responsible for freeing *ppFree if it is non-NULL in order
+** to reclaim memory.
+*/
+static const char *escapeOutput(
+  ShellState *p,
+  const char *zInX,
+  char **ppFree
+){
+  i64 i, j;
+  i64 nCtrl = 0;
+  unsigned char *zIn;
+  unsigned char c;
+  unsigned char *zOut;
+
+
+  /* No escaping if disabled */
+  if( p->eEscMode==SHELL_ESC_OFF ){
+    *ppFree = 0;
+     return zInX;
+  }
+
+  /* Count the number of control characters in the string. */
+  zIn = (unsigned char*)zInX;
+  for(i=0; (c = zIn[i])!=0; i++){
+    if( c<=0x1f
+     && c!='\t'
+     && c!='\n'
+     && (c!='\r' || zIn[i+1]!='\n')
+    ){
+      nCtrl++;
+    }
+  }
+  if( nCtrl==0 ){
+    *ppFree = 0;
+    return zInX;
+  }
+  zOut = sqlite3_malloc64( i + 2*nCtrl + 1 );
+  shell_check_oom(zOut);
+  for(i=j=0; (c = zIn[i])!=0; i++){
+    if( c>0x1f
+     || c=='\t'
+     || c=='\n'
+     || (c=='\r' && zIn[i+1]=='\n')
+    ){
+      continue;
+    }
+    if( i>0 ){
+      memcpy(&zOut[j], zIn, i);
+      j += i;
+    }
+    zIn += i+1;
+    i = -1;
+    zOut[j++] = 0xe2;
+    zOut[j++] = 0x90;
+    zOut[j++] = 0x80+c;
+  }
+  if( i>0 ){
+    memcpy(&zOut[j], zIn, i);
+    j += i;
+  }
+  zOut[j] = 0;
+  *ppFree = (char*)zOut;
+  return (char*)zOut;
+}
+
+/*
 ** Output the given string with characters that are special to
 ** HTML escaped.
 */
@@ -2597,8 +2681,12 @@ static int shell_callback(
       }
       if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out);
       for(i=0; i<nArg; i++){
+        char *pFree = 0;
+        const char *pDisplay;
+        pDisplay = escapeOutput(p, azArg[i] ? azArg[i] : p->nullValue, &pFree);
         sqlite3_fprintf(p->out, "%*s = %s%s", w, azCol[i],
-              azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator);
+                        pDisplay, p->rowSeparator);
+        if( pFree ) sqlite3_free(pFree);
       }
       break;
     }
@@ -2730,15 +2818,23 @@ static int shell_callback(
     case MODE_List: {
       if( p->cnt++==0 && p->showHeader ){
         for(i=0; i<nArg; i++){
-          sqlite3_fprintf(p->out, "%s%s", azCol[i],
+          char *z = azCol[i];
+          char *pFree;
+          const char *zOut = escapeOutput(p, z, &pFree);
+          sqlite3_fprintf(p->out, "%s%s", zOut,
                           i==nArg-1 ? p->rowSeparator : p->colSeparator);
+          if( pFree ) sqlite3_free(pFree);
         }
       }
       if( azArg==0 ) break;
       for(i=0; i<nArg; i++){
         char *z = azArg[i];
+        char *pFree;
+        const char *zOut;
         if( z==0 ) z = p->nullValue;
-        sqlite3_fputs(z, p->out);
+        zOut = escapeOutput(p, z, &pFree);
+        sqlite3_fputs(zOut, p->out);
+        if( pFree ) sqlite3_free(pFree);
         sqlite3_fputs((i<nArg-1)? p->colSeparator : p->rowSeparator, p->out);
       }
       break;
@@ -3891,6 +3987,7 @@ static char *translateForDisplayAndDup(
       j++;
       continue;
     }
+    if( c==0 || c=='\n' || (c=='\r' && z[i+1]=='\n') ) break;
     if( c=='\t' ){
       do{
         n++;
@@ -3899,7 +3996,9 @@ static char *translateForDisplayAndDup(
       i++;
       continue;
     }
-    break;
+    n++;
+    j += 3;
+    i++;
   }
   if( n>=mxWidth && bWordWrap  ){
     /* Perhaps try to back up to a better place to break the line */
@@ -3946,6 +4045,7 @@ static char *translateForDisplayAndDup(
       zOut[j++] = z[i++];
       continue;
     }
+    if( c==0 ) break;
     if( z[i]=='\t' ){
       do{
         n++;
@@ -3954,7 +4054,10 @@ static char *translateForDisplayAndDup(
       i++;
       continue;
     }
-    break;
+    zOut[j++] = 0xe2;
+    zOut[j++] = 0x90;
+    zOut[j++] = 0x80 + c;
+    i++;
   }
   zOut[j] = 0;
   return (char*)zOut;
@@ -9792,6 +9895,10 @@ static int do_meta_command(char *zLine, ShellState *p){
         cmOpts.bQuote = 1;
       }else if( optionMatch(z,"noquote") ){
         cmOpts.bQuote = 0;
+      }else if( optionMatch(z,"escape") ){
+        p->eEscMode = SHELL_ESC_GRAPHIC;
+      }else if( optionMatch(z,"noescape") ){
+        p->eEscMode = SHELL_ESC_OFF;
       }else if( zMode==0 ){
         zMode = z;
         /* Apply defaults for qbox pseudo-mode.  If that
@@ -9825,13 +9932,19 @@ static int do_meta_command(char *zLine, ShellState *p){
        || (p->mode>=MODE_Markdown && p->mode<=MODE_Box)
       ){
         sqlite3_fprintf(p->out,
-              "current output mode: %s --wrap %d --wordwrap %s --%squote\n",
+              "current output mode: %s --wrap %d --wordwrap %s "
+              "--%squote --%sescape\n",
               modeDescr[p->mode], p->cmOpts.iWrap,
               p->cmOpts.bWordWrap ? "on" : "off",
-              p->cmOpts.bQuote ? "" : "no");
+              p->cmOpts.bQuote ? "" : "no",
+              p->eEscMode ? "no" : ""
+        );
       }else{
         sqlite3_fprintf(p->out,
-              "current output mode: %s\n", modeDescr[p->mode]);
+              "current output mode: %s --%sescape\n",
+              modeDescr[p->mode],
+              p->eEscMode ? "no" : ""
+        );
       }
       zMode = modeDescr[p->mode];
     }
@@ -12586,6 +12699,7 @@ static const char zOptions[] =
   "   -deserialize         open the database using sqlite3_deserialize()\n"
 #endif
   "   -echo                print inputs before execution\n"
+  "   -escape              print control character XX as U+24XX\n"
   "   -init FILENAME       read/process named file\n"
   "   -[no]header          turn headers on or off\n"
 #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
@@ -12608,6 +12722,7 @@ static const char zOptions[] =
   "   -multiplex           enable the multiplexor VFS\n"
 #endif
   "   -newline SEP         set output row separator. Default: '\\n'\n"
+  "   -noescape            output control characters unmodified\n"
   "   -nofollow            refuse to open symbolic links to database files\n"
   "   -nonce STRING        set the safe-mode escape nonce\n"
   "   -no-rowid-in-view    Disable rowid-in-view using sqlite3_config()\n"
@@ -13118,6 +13233,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-csv")==0 ){
       data.mode = MODE_Csv;
       memcpy(data.colSeparator,",",2);
+    }else if( cli_strcmp(z,"-noescape")==0 ){
+      data.eEscMode = SHELL_ESC_OFF;
+    }else if( cli_strcmp(z,"-escape")==0 ){
+      data.eEscMode = SHELL_ESC_GRAPHIC;
 #ifdef SQLITE_HAVE_ZLIB
     }else if( cli_strcmp(z,"-zip")==0 ){
       data.openMode = SHELL_OPEN_ZIPFILE;
diff --git a/test/shell1.test b/test/shell1.test
index a272295f5..6189ff83a 100644
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -450,7 +450,7 @@ do_test shell1-3.12.3 {
 #                          tcl      TCL list elements
 do_test shell1-3.13.1 {
   catchcmd "test.db" ".mode"
-} {0 {current output mode: list}}
+} {0 {current output mode: list --escape}}
 do_test shell1-3.13.2 {
   catchcmd "test.db" ".mode FOO"
 } {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}}