improve raw strings

jeff-hykin · Apr 16, 2024 · 1babbb7 · 1babbb7
1 parent 47e2807
commit 1babbb7
Show file tree

Hide file tree

Showing 8 changed files with 90 additions and 64 deletions.
diff --git a/autogenerated/cpp.embedded.macro.tmLanguage.json b/autogenerated/cpp.embedded.macro.tmLanguage.json
diff --git a/autogenerated/cpp.tmLanguage.json b/autogenerated/cpp.tmLanguage.json
@@ -14337,11 +14337,33 @@
             }
           ]
         },
+        {
+          "begin": "((?:[uUL]8?)?R)\\\"\\(",
+          "end": "\\)\\\"",
+          "beginCaptures": {
+            "0": {
+              "name": "punctuation.definition.string.begin.cpp"
+            },
+            "1": {
+              "name": "meta.encoding.cpp"
+            }
+          },
+          "endCaptures": {
+            "0": {
+              "name": "punctuation.definition.string.end.cpp"
+            }
+          },
+          "name": "string.quoted.double.raw.cpp",
+          "patterns": [
+            {
+            }
+          ]
+        },
         {
           "begin": "((?:u|u8|U|L)?R)\"(?:([^ ()\\\\\\t]{0,16})|([^ ()\\\\\\t]*))\\(",
           "beginCaptures": {
             "0": {
-              "name": "punctuation.definition.string.begin"
+              "name": "punctuation.definition.string.$2.begin"
             },
             "1": {
               "name": "meta.encoding"
@@ -14350,22 +14372,22 @@
               "name": "invalid.illegal.delimiter-too-long"
             }
           },
-          "end": "(\\)\\2(\\3)\")(?:((?:[a-zA-Z]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)|(_(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*))?",
+          "end": "(\\)(\\2)(\\3)\")(?:((?:[a-zA-Z]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)|(_(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*))?",
           "endCaptures": {
             "1": {
-              "name": "punctuation.definition.string.end"
+              "name": "punctuation.definition.string.$2.end"
             },
-            "2": {
+            "3": {
               "name": "invalid.illegal.delimiter-too-long"
             },
-            "3": {
+            "4": {
               "name": "keyword.other.suffix.literal.user-defined.reserved.string.cpp"
             },
-            "4": {
+            "5": {
               "name": "keyword.other.suffix.literal.user-defined.string.cpp"
             }
           },
-          "name": "string.quoted.double.raw"
+          "name": "string.quoted.double.raw.$2"
         }
       ]
     },

diff --git a/autogenerated/cpp_scopes.txt b/autogenerated/cpp_scopes.txt
@@ -287,10 +287,10 @@ punctuation.definition.parameters.begin.lambda.cpp
 punctuation.definition.parameters.begin.preprocessor.cpp
 punctuation.definition.parameters.end.lambda.cpp
 punctuation.definition.parameters.end.preprocessor.cpp
-punctuation.definition.string.begin
+punctuation.definition.string.$2.begin
+punctuation.definition.string.$2.end
 punctuation.definition.string.begin.assembly.cpp
 punctuation.definition.string.begin.cpp
-punctuation.definition.string.end
 punctuation.definition.string.end.assembly.cpp
 punctuation.definition.string.end.cpp
 punctuation.section.angle-brackets.begin.template.call.cpp
@@ -464,7 +464,8 @@ storage.type.union.declare.cpp
 storage.type.union.parameter.cpp
 string.quoted.double.cpp
 string.quoted.double.include.cpp
-string.quoted.double.raw
+string.quoted.double.raw.$2
+string.quoted.double.raw.cpp
 string.quoted.double.raw.regex.cpp
 string.quoted.other.lt-gt.include.cpp
 string.quoted.single.cpp

diff --git a/language_examples/#610.spec.yaml b/language_examples/#610.spec.yaml
@@ -382,22 +382,22 @@
     - keyword.operator.assignment
 - source: LR
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw.--
+    - punctuation.definition.string.--.begin
   scopes:
     - meta.encoding
 - source: '"--('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string.--.begin
 - source: rws
 - source: )--"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string.--.end
 - source: _wrap
   scopes:
     - keyword.other.suffix.literal.user-defined.string
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw.--
 - source: ;
   scopes:
     - punctuation.terminator.statement

diff --git a/language_examples/feature_tagged_strings.spec.yaml b/language_examples/feature_tagged_strings.spec.yaml
@@ -8,19 +8,19 @@
     - keyword.operator.assignment
 - source: R
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw._r
+    - punctuation.definition.string._r.begin
   scopes:
     - meta.encoding
 - source: '"_r('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string._r.begin
 - source: '[0-9''"\+]+'
 - source: )_r"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string._r.end
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw._r
 - source: ;
   scopes:
     - punctuation.terminator.statement
@@ -34,19 +34,19 @@
     - keyword.operator.assignment
 - source: R
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw.re
+    - punctuation.definition.string.re.begin
   scopes:
     - meta.encoding
 - source: '"re('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string.re.begin
 - source: '[0-9''"\+]+'
 - source: )re"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string.re.end
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw.re
 - source: ;
   scopes:
     - punctuation.terminator.statement
@@ -60,19 +60,19 @@
     - keyword.operator.assignment
 - source: R
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw.regex
+    - punctuation.definition.string.regex.begin
   scopes:
     - meta.encoding
 - source: '"regex('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string.regex.begin
 - source: '[0-9''"\+]+'
 - source: )regex"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string.regex.end
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw.regex
 - source: ;
   scopes:
     - punctuation.terminator.statement
diff --git a/language_examples/ranged_for.spec.yaml b/language_examples/ranged_for.spec.yaml
@@ -62545,13 +62545,13 @@
     - keyword.operator.assignment
 - source: R
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw.s
+    - punctuation.definition.string.s.begin
   scopes:
     - meta.encoding
 - source: '"s('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string.s.begin
 - source: '      Returns a list if the [extended POSIX regular'
 - source: '      expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)'
 - source: '      *regex* matches *str* precisely, otherwise returns `null`. Each item'
@@ -62574,9 +62574,9 @@
 - source: '      Evaluates to `[ "FOO" ]`.'
 - source: )s"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string.s.end
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw.s
 - source: ','
   scopes:
     - punctuation.separator.delimiter.comma
@@ -64293,13 +64293,13 @@
     - keyword.operator.assignment
 - source: R
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw.s
+    - punctuation.definition.string.s.begin
   scopes:
     - meta.encoding
 - source: '"s('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string.s.begin
 - source: '      Returns a list composed of non matched strings interleaved with the'
 - source: '      lists of the [extended POSIX regular'
 - source: '      expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04)'
@@ -64323,9 +64323,9 @@
 - source: '      Evaluates to `[ " " [ "FOO" ] " " ]`.'
 - source: )s"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string.s.end
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw.s
 - source: ','
   scopes:
     - punctuation.separator.delimiter.comma

diff --git a/language_examples/raw_strings.spec.yaml b/language_examples/raw_strings.spec.yaml
@@ -38,13 +38,13 @@
     - punctuation.section.block.begin.bracket.curly
 - source: R
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw.glsl
+    - punctuation.definition.string.glsl.begin
   scopes:
     - meta.encoding
 - source: '"glsl('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string.glsl.begin
 - source: '        precision highp float;'
 - source: '        attribute vec2 vPosition;'
 - source: '        attribute vec3 textureCoord;'
@@ -56,9 +56,9 @@
 - source: '        }'
 - source: )glsl"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string.glsl.end
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw.glsl
 - source: '}'
   scopes:
     - punctuation.section.block.end.bracket.curly
@@ -82,19 +82,19 @@
     - punctuation.section.block.begin.bracket.curly
 - source: R
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw.sql
+    - punctuation.definition.string.sql.begin
   scopes:
     - meta.encoding
 - source: '"sql('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string.sql.begin
 - source: '        select a, b, c from someTable where a > 10 order by a asc'
 - source: )sql"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string.sql.end
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw.sql
 - source: '}'
   scopes:
     - punctuation.section.block.end.bracket.curly
@@ -118,19 +118,19 @@
     - punctuation.section.block.begin.bracket.curly
 - source: R
   scopesBegin:
-    - string.quoted.double.raw
-    - punctuation.definition.string.begin
+    - string.quoted.double.raw.re
+    - punctuation.definition.string.re.begin
   scopes:
     - meta.encoding
 - source: '"re('
   scopesEnd:
-    - punctuation.definition.string.begin
+    - punctuation.definition.string.re.begin
 - source: '        ((?:[uUL]8?)?R)\\\"(?:(?:_r|re)|regex)\\()'
 - source: )re"
   scopes:
-    - punctuation.definition.string.end
+    - punctuation.definition.string.re.end
   scopesEnd:
-    - string.quoted.double.raw
+    - string.quoted.double.raw.re
 - source: '}'
   scopes:
     - punctuation.section.block.end.bracket.curly

diff --git a/main/patterns/raw_strings.rb b/main/patterns/raw_strings.rb
@@ -3,6 +3,7 @@ def generateTaggedRawString(name, tag_pattern, inner_pattern, tag_as_meta:false)
     # (because they're part of another language or syntax)
     meta = ""
     meta = "meta." if tag_as_meta
+    name = ".#{name}" if name
     return PatternRange.new(
         start_pattern: Pattern.new(
             match: Pattern.new(
@@ -15,7 +16,7 @@ def generateTaggedRawString(name, tag_pattern, inner_pattern, tag_as_meta:false)
             match: Pattern.new(")").then(tag_pattern).then(/\"/),
             tag_as: "punctuation.definition.string.end"
         ),
-        tag_as: meta+"string.quoted.double.raw.#{name}",
+        tag_as: meta+"string.quoted.double.raw#{name}",
         includes: [
             inner_pattern,
         ]
@@ -28,7 +29,7 @@ def getRawStringPatterns()
         begin: "((?:u|u8|U|L)?R)\"(?:([^ ()\\\\\\t]{0,16})|([^ ()\\\\\\t]*))\\(",
         beginCaptures: {
             "0" => {
-                name: "punctuation.definition.string.begin"
+                name: "punctuation.definition.string.$2.begin"
             },
             "1" => {
                 name: "meta.encoding"
@@ -37,30 +38,32 @@ def getRawStringPatterns()
                 name: "invalid.illegal.delimiter-too-long"
             }
         },
-        end: "(\\)\\2(\\3)\")(?:((?:[a-zA-Z]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)|(_(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*))?",
+        end: "(\\)(\\2)(\\3)\")(?:((?:[a-zA-Z]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*)|(_(?:[a-zA-Z0-9_]|(?:\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}))*))?",
         endCaptures: {
             "1" => {
-                name: "punctuation.definition.string.end"
+                name: "punctuation.definition.string.$2.end"
             },
-            "2" => {
+            "3" => {
                 name: "invalid.illegal.delimiter-too-long"
             },
-            "3" => {
+            "4" => {
                 name: "keyword.other.suffix.literal.user-defined.reserved.string.cpp"
             },
-            "4" => {
+            "5" => {
                 name: "keyword.other.suffix.literal.user-defined.string.cpp"
             },
         },
-        name: "string.quoted.double.raw"
+        name: "string.quoted.double.raw.$2"
     )
     regex = generateTaggedRawString("regex", Pattern.new(/_r/).or(/re/).or(/regex/), "source.regexp.python")
     sql = generateTaggedRawString("sql", Pattern.new(/[pP]?(?:sql|SQL)/).or(/d[dm]l/), "source.sql", tag_as_meta: true)
     glsl = generateTaggedRawString("glsl", Pattern.new(/glsl/).or(/GLSL/), "source.glsl", tag_as_meta: true)
+    plain = generateTaggedRawString(nil, Pattern.new(//), "", tag_as_meta: false)
     return [
         regex,
         glsl,
         sql,
+        plain,
         default,
     ]
 end