aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTom Lane <tgl@sss.pgh.pa.us>2024-09-05 12:42:33 -0400
committerTom Lane <tgl@sss.pgh.pa.us>2024-09-05 12:42:33 -0400
commit7dcbf0afa28cc8c3dd83bef0428a18ac177f129e (patch)
tree513c9b329c82b930c2c4f6d16ef74b1aec9d29de
parent07b828e9d4aa916f1763774787440d914eea69c4 (diff)
downloadpostgresql-7dcbf0afa28cc8c3dd83bef0428a18ac177f129e.tar.gz
postgresql-7dcbf0afa28cc8c3dd83bef0428a18ac177f129e.zip
Prevent mis-encoding of "trailing junk after numeric literal" errors.
Since commit 2549f0661, we reject an identifier immediately following a numeric literal (without separating whitespace), because that risks ambiguity with hex/octal/binary integers. However, that patch used token patterns like "{integer}{ident_start}", which is problematic because {ident_start} matches only a single byte. If the first character after the integer is a multibyte character, this ends up with flex reporting an error message that includes a partial multibyte character. That can cause assorted bad-encoding problems downstream, both in the report to the client and in the postmaster log file. To fix, use {identifier} not {ident_start} in the "junk" token patterns, so that they will match complete multibyte characters. This seems generally better user experience quite aside from the encoding problem: for "123abc" the error message will now say that the error appeared at or near "123abc" instead of "123a". While at it, add some commentary about why these patterns exist and how they work. Report and patch by Karina Litskevich; review by Pavel Borisov. Back-patch to v15 where the problem came in. Discussion: https://postgr.es/m/CACiT8iZ_diop=0zJ7zuY3BXegJpkKK1Av-PU7xh0EDYHsa5+=g@mail.gmail.com
-rw-r--r--src/backend/parser/scan.l44
-rw-r--r--src/fe_utils/psqlscan.l41
-rw-r--r--src/interfaces/ecpg/preproc/pgc.l41
-rw-r--r--src/test/regress/expected/numerology.out10
4 files changed, 74 insertions, 62 deletions
diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index 3248fb51080..c97db946110 100644
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -412,16 +412,30 @@ numericfail {decinteger}\.\.
real ({decinteger}|{numeric})[Ee][-+]?{decinteger}
realfail ({decinteger}|{numeric})[Ee][-+]
-decinteger_junk {decinteger}{ident_start}
-hexinteger_junk {hexinteger}{ident_start}
-octinteger_junk {octinteger}{ident_start}
-bininteger_junk {bininteger}{ident_start}
-numeric_junk {numeric}{ident_start}
-real_junk {real}{ident_start}
-
/* Positional parameters don't accept underscores. */
param \${decdigit}+
-param_junk \${decdigit}+{ident_start}
+
+/*
+ * An identifier immediately following an integer literal is disallowed because
+ * in some cases it's ambiguous what is meant: for example, 0x1234 could be
+ * either a hexinteger or a decinteger "0" and an identifier "x1234". We can
+ * detect such problems by seeing if integer_junk matches a longer substring
+ * than any of the XXXinteger patterns (decinteger, hexinteger, octinteger,
+ * bininteger). One "junk" pattern is sufficient because
+ * {decinteger}{identifier} will match all the same strings we'd match with
+ * {hexinteger}{identifier} etc.
+ *
+ * Note that the rule for integer_junk must appear after the ones for
+ * XXXinteger to make this work correctly: 0x1234 will match both hexinteger
+ * and integer_junk, and we need hexinteger to be chosen in that case.
+ *
+ * Also disallow strings matched by numeric_junk, real_junk and param_junk
+ * for consistency.
+ */
+integer_junk {decinteger}{identifier}
+numeric_junk {numeric}{identifier}
+real_junk {real}{identifier}
+param_junk \${decdigit}+{identifier}
other .
@@ -1049,19 +1063,7 @@ other .
SET_YYLLOC();
yyerror("trailing junk after numeric literal");
}
-{decinteger_junk} {
- SET_YYLLOC();
- yyerror("trailing junk after numeric literal");
- }
-{hexinteger_junk} {
- SET_YYLLOC();
- yyerror("trailing junk after numeric literal");
- }
-{octinteger_junk} {
- SET_YYLLOC();
- yyerror("trailing junk after numeric literal");
- }
-{bininteger_junk} {
+{integer_junk} {
SET_YYLLOC();
yyerror("trailing junk after numeric literal");
}
diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l
index ddc4658b925..8e8b049e15f 100644
--- a/src/fe_utils/psqlscan.l
+++ b/src/fe_utils/psqlscan.l
@@ -348,16 +348,30 @@ numericfail {decinteger}\.\.
real ({decinteger}|{numeric})[Ee][-+]?{decinteger}
realfail ({decinteger}|{numeric})[Ee][-+]
-decinteger_junk {decinteger}{ident_start}
-hexinteger_junk {hexinteger}{ident_start}
-octinteger_junk {octinteger}{ident_start}
-bininteger_junk {bininteger}{ident_start}
-numeric_junk {numeric}{ident_start}
-real_junk {real}{ident_start}
-
/* Positional parameters don't accept underscores. */
param \${decdigit}+
-param_junk \${decdigit}+{ident_start}
+
+/*
+ * An identifier immediately following an integer literal is disallowed because
+ * in some cases it's ambiguous what is meant: for example, 0x1234 could be
+ * either a hexinteger or a decinteger "0" and an identifier "x1234". We can
+ * detect such problems by seeing if integer_junk matches a longer substring
+ * than any of the XXXinteger patterns (decinteger, hexinteger, octinteger,
+ * bininteger). One "junk" pattern is sufficient because
+ * {decinteger}{identifier} will match all the same strings we'd match with
+ * {hexinteger}{identifier} etc.
+ *
+ * Note that the rule for integer_junk must appear after the ones for
+ * XXXinteger to make this work correctly: 0x1234 will match both hexinteger
+ * and integer_junk, and we need hexinteger to be chosen in that case.
+ *
+ * Also disallow strings matched by numeric_junk, real_junk and param_junk
+ * for consistency.
+ */
+integer_junk {decinteger}{identifier}
+numeric_junk {numeric}{identifier}
+real_junk {real}{identifier}
+param_junk \${decdigit}+{identifier}
/* psql-specific: characters allowed in variable names */
variable_char [A-Za-z\200-\377_0-9]
@@ -898,16 +912,7 @@ other .
{realfail} {
ECHO;
}
-{decinteger_junk} {
- ECHO;
- }
-{hexinteger_junk} {
- ECHO;
- }
-{octinteger_junk} {
- ECHO;
- }
-{bininteger_junk} {
+{integer_junk} {
ECHO;
}
{numeric_junk} {
diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l
index f9d68a96e79..ba0db561457 100644
--- a/src/interfaces/ecpg/preproc/pgc.l
+++ b/src/interfaces/ecpg/preproc/pgc.l
@@ -381,16 +381,30 @@ numericfail {decinteger}\.\.
real ({decinteger}|{numeric})[Ee][-+]?{decinteger}
realfail ({decinteger}|{numeric})[Ee][-+]
-decinteger_junk {decinteger}{ident_start}
-hexinteger_junk {hexinteger}{ident_start}
-octinteger_junk {octinteger}{ident_start}
-bininteger_junk {bininteger}{ident_start}
-numeric_junk {numeric}{ident_start}
-real_junk {real}{ident_start}
-
/* Positional parameters don't accept underscores. */
param \${decdigit}+
-param_junk \${decdigit}+{ident_start}
+
+/*
+ * An identifier immediately following an integer literal is disallowed because
+ * in some cases it's ambiguous what is meant: for example, 0x1234 could be
+ * either a hexinteger or a decinteger "0" and an identifier "x1234". We can
+ * detect such problems by seeing if integer_junk matches a longer substring
+ * than any of the XXXinteger patterns (decinteger, hexinteger, octinteger,
+ * bininteger). One "junk" pattern is sufficient because
+ * {decinteger}{identifier} will match all the same strings we'd match with
+ * {hexinteger}{identifier} etc.
+ *
+ * Note that the rule for integer_junk must appear after the ones for
+ * XXXinteger to make this work correctly: 0x1234 will match both hexinteger
+ * and integer_junk, and we need hexinteger to be chosen in that case.
+ *
+ * Also disallow strings matched by numeric_junk, real_junk and param_junk
+ * for consistency.
+ */
+integer_junk {decinteger}{identifier}
+numeric_junk {numeric}{identifier}
+real_junk {real}{identifier}
+param_junk \${decdigit}+{identifier}
/* special characters for other dbms */
/* we have to react differently in compat mode */
@@ -993,16 +1007,7 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+
* Note that some trailing junk is valid in C (such as 100LL), so we
* contain this to SQL mode.
*/
-{decinteger_junk} {
- mmfatal(PARSE_ERROR, "trailing junk after numeric literal");
- }
-{hexinteger_junk} {
- mmfatal(PARSE_ERROR, "trailing junk after numeric literal");
- }
-{octinteger_junk} {
- mmfatal(PARSE_ERROR, "trailing junk after numeric literal");
- }
-{bininteger_junk} {
+{integer_junk} {
mmfatal(PARSE_ERROR, "trailing junk after numeric literal");
}
{numeric_junk} {
diff --git a/src/test/regress/expected/numerology.out b/src/test/regress/expected/numerology.out
index 8d4a3ba228a..3512a1d04ce 100644
--- a/src/test/regress/expected/numerology.out
+++ b/src/test/regress/expected/numerology.out
@@ -171,7 +171,7 @@ SELECT -0x8000000000000001;
-- error cases
SELECT 123abc;
-ERROR: trailing junk after numeric literal at or near "123a"
+ERROR: trailing junk after numeric literal at or near "123abc"
LINE 1: SELECT 123abc;
^
SELECT 0x0o;
@@ -318,7 +318,7 @@ ERROR: trailing junk after numeric literal at or near "100_"
LINE 1: SELECT 100_;
^
SELECT 100__000;
-ERROR: trailing junk after numeric literal at or near "100_"
+ERROR: trailing junk after numeric literal at or near "100__000"
LINE 1: SELECT 100__000;
^
SELECT _1_000.5;
@@ -330,7 +330,7 @@ ERROR: trailing junk after numeric literal at or near "1_000_"
LINE 1: SELECT 1_000_.5;
^
SELECT 1_000._5;
-ERROR: trailing junk after numeric literal at or near "1_000._"
+ERROR: trailing junk after numeric literal at or near "1_000._5"
LINE 1: SELECT 1_000._5;
^
SELECT 1_000.5_;
@@ -338,11 +338,11 @@ ERROR: trailing junk after numeric literal at or near "1_000.5_"
LINE 1: SELECT 1_000.5_;
^
SELECT 1_000.5e_1;
-ERROR: trailing junk after numeric literal at or near "1_000.5e"
+ERROR: trailing junk after numeric literal at or near "1_000.5e_1"
LINE 1: SELECT 1_000.5e_1;
^
PREPARE p1 AS SELECT $0_1;
-ERROR: trailing junk after parameter at or near "$0_"
+ERROR: trailing junk after parameter at or near "$0_1"
LINE 1: PREPARE p1 AS SELECT $0_1;
^
--