feat(compiler): tokenize interpolations in escapable raw text

Expose interpolation boundaries in textarea and title so consumers can
replace regular expressions with lexer tokens. Preserve RCDATA entity
validation and closing-tag handling, and add public parser regressions.

Author: splincode

packages/angular-html-parser/readme.md

@@ -97,6 +97,7 @@ interface Options {
 - support [bogus comments](https://www.w3.org/TR/html5/syntax.html#bogus-comment-state) (`<!...>`, `<?...>`)
 - ~~support full [named entities](https://html.spec.whatwg.org/multipage/entities.json)~~ (fixed upstream)
 - add `type` property to nodes
+- tokenize interpolation expressions inside escapable raw text (`textarea` and HTML `title`)
 - value span for attributes includes quotes
 
 ## Development

packages/angular-html-parser/test/index_spec.ts

@@ -298,3 +298,127 @@ describe("public token API", () => {
     expect(token.parts).toEqual(["{{", ' "}}" ', "}}"]);
   });
 });
+
+describe("escapable raw text interpolation", () => {
+  describe.each(["textarea", "title"])("<%s>", (tagName) => {
+    it.each([
+      {
+        name: "multiple expressions",
+        content: "before {{value}} after {{other}}",
+        value: "before {{value}} after {{other}}",
+        expressions: ["value", "other"],
+      },
+      {
+        name: "double braces inside a string",
+        content: 'before {{ "{{first}} / {{second}}" }} after',
+        value: 'before {{ "{{first}} / {{second}}" }} after',
+        expressions: [' "{{first}} / {{second}}" '],
+      },
+      {
+        name: "literal markup inside and outside an expression",
+        content: '<b>{{ "<span>" }}</b>',
+        value: '<b>{{ "<span>" }}</b>',
+        expressions: [' "<span>" '],
+      },
+      {
+        name: "entities inside and outside an expression",
+        content: '&lt;{{ "&amp;" }}&#32;{{value}}&gt;',
+        value: '<{{ "&" }} {{value}}>',
+        expressions: [' "&amp;" ', "value"],
+      },
+      {
+        name: "a leading LF",
+        content: "\n{{value}}\n",
+        value: "\n{{value}}\n",
+        expressions: ["value"],
+      },
+      {
+        name: "CRLF inside an expression",
+        content: "{{a\r\n+b}}tail",
+        value: "{{a\n+b}}tail",
+        expressions: ["a\n+b"],
+      },
+      {
+        name: "adjacent and empty expressions",
+        content: "{{one}}{{two}}{{}}",
+        value: "{{one}}{{two}}{{}}",
+        expressions: ["one", "two", ""],
+      },
+    ])("should tokenize $name", ({ content, value, expressions }) => {
+      const result = parse(`<${tagName}>${content}</${tagName}>`);
+      expect(result.errors).toEqual([]);
+      const element = result.rootNodes[0] as ast.Element;
+      expect(element.children).toHaveLength(1);
+      const text = element.children[0] as ast.Text;
+      expect(text.value).toBe(
+        tagName === "textarea" && value.startsWith("\n")
+          ? value.slice(1)
+          : value,
+      );
+      expect(
+        text.tokens
+          .filter((token) => token.type === TokenType.INTERPOLATION)
+          .map((token) => token.parts),
+      ).toEqual(expressions.map((expression) => ["{{", expression, "}}"]));
+      expect(
+        text.tokens.map((token) => token.sourceSpan.toString()).join(""),
+      ).toBe(content);
+    });
+  });
+
+  it("should stop an unfinished interpolation at the closing tag", () => {
+    const result = parse("<textarea>{{value</textarea><div>after</div>");
+    expect(result.errors).toEqual([]);
+    const element = result.rootNodes[0] as ast.Element;
+    const text = element.children[0] as ast.Text;
+    expect(text.value).toBe("{{value");
+    expect(
+      text.tokens.find((token) => token.type === TokenType.INTERPOLATION)
+        ?.parts,
+    ).toEqual(["{{", "value"]);
+    expect(element.endSourceSpan?.toString()).toBe("</textarea>");
+    expect((result.rootNodes[1] as ast.Element).name).toBe("div");
+  });
+
+  it.each(["", "\\"])(
+    "should keep the closing tag significant after %j in a quoted expression",
+    (escape) => {
+      const result = parse(
+        `<textarea>{{ "${escape}</textarea>" }}<div>after</div>`,
+      );
+      expect(result.errors).toEqual([]);
+      const element = result.rootNodes[0] as ast.Element;
+      expect((element.children[0] as ast.Text).value).toBe(`{{ "${escape}`);
+      expect(element.endSourceSpan?.toString()).toBe("</textarea>");
+      expect((result.rootNodes[2] as ast.Element).name).toBe("div");
+    },
+  );
+
+  describe.each(["textarea", "title"])(
+    "invalid entities in <%s>",
+    (tagName) => {
+      it.each([
+        { entity: "&bogus;", error: "Unknown entity" },
+        { entity: "&#x110000;", error: "Unknown entity" },
+        { entity: "&#x;", error: "Unknown entity" },
+        { entity: "&#x41", error: "Unable to parse entity" },
+        { entity: "\\&bogus;", error: "Unknown entity" },
+      ])("should report $entity as a parse error", ({ entity, error }) => {
+        const result = parse(`<${tagName}>{{ "${entity}" }}</${tagName}>`);
+        expect(result.errors[0]?.msg).toContain(error);
+      });
+    },
+  );
+
+  it.each(["script", "style"])("should keep <%s> as raw text", (tagName) => {
+    const content = '{{ "<b>&amp;" }}';
+    const result = parse(`<${tagName}>${content}</${tagName}>`);
+    expect(result.errors).toEqual([]);
+    const element = result.rootNodes[0] as ast.Element;
+    const text = element.children[0] as ast.Text;
+    expect(text.value).toBe(content);
+    expect(text.tokens.map((token) => token.type)).toEqual([
+      TokenType.RAW_TEXT,
+    ]);
+  });
+});

packages/compiler/src/ml_parser/lexer.ts

@@ -738,6 +738,16 @@ class _Tokenizer {
 
   private _consumeEntity(textTokenType: TokenType): void {
     this._beginToken(TokenType.ENCODED_ENTITY);
+    const start = this._cursor.clone();
+    const parts = this._readEntity();
+    if (parts.length === 1) {
+      this._beginToken(textTokenType, start);
+    }
+    this._endToken(parts);
+  }
+
+  /** Consume an HTML entity without emitting a token. */
+  private _readEntity(): [string] | [string, string] {
     const start = this._cursor.clone();
     this._cursor.advance();
     if (this._attemptCharCode(chars.$HASH)) {
@@ -758,7 +768,7 @@ class _Tokenizer {
       this._cursor.advance();
       try {
         const charCode = parseInt(strNum, isHex ? 16 : 10);
-        this._endToken([String.fromCodePoint(charCode), this._cursor.getChars(start)]);
+        return [String.fromCodePoint(charCode), this._cursor.getChars(start)];
       } catch {
         throw this._createError(
           _unknownEntityErrorMsg(this._cursor.getChars(start)),
@@ -769,25 +779,39 @@ class _Tokenizer {
       const nameStart = this._cursor.clone();
       this._attemptCharCodeUntilFn(isNamedEntityEnd);
       if (this._cursor.peek() != chars.$SEMICOLON) {
-        // No semicolon was found so abort the encoded entity token that was in progress, and treat
-        // this as a text token
-        this._beginToken(textTokenType, start);
+        // Without a semicolon, only the ampersand is consumed as plain text.
         this._cursor = nameStart;
-        this._endToken(['&']);
+        return ['&'];
       } else {
         const name = this._cursor.getChars(nameStart);
         this._cursor.advance();
         const char = Object.hasOwn(NAMED_ENTITIES, name) && NAMED_ENTITIES[name];
         if (!char) {
           throw this._createError(_unknownEntityErrorMsg(name), this._cursor.getSpan(start));
         }
-        this._endToken([char, `&${name};`]);
+        return [char, `&${name};`];
       }
     }
   }
 
   private _consumeRawText(consumeEntities: boolean, endMarkerPredicate: () => boolean): void {
-    this._beginToken(consumeEntities ? TokenType.ESCAPABLE_RAW_TEXT : TokenType.RAW_TEXT);
+    if (consumeEntities) {
+      const isRawTextEnd = () => {
+        const start = this._cursor.clone();
+        const foundEndMarker = endMarkerPredicate();
+        this._cursor = start;
+        return foundEndMarker;
+      };
+      this._consumeWithInterpolation(
+        TokenType.ESCAPABLE_RAW_TEXT,
+        TokenType.INTERPOLATION,
+        isRawTextEnd,
+        isRawTextEnd,
+      );
+      return;
+    }
+
+    this._beginToken(TokenType.RAW_TEXT);
     const parts: string[] = [];
     while (true) {
       const tagCloseStart = this._cursor.clone();
@@ -796,14 +820,7 @@ class _Tokenizer {
       if (foundEndMarker) {
         break;
       }
-      if (consumeEntities && this._cursor.peek() === chars.$AMPERSAND) {
-        this._endToken([this._processCarriageReturns(parts.join(''))]);
-        parts.length = 0;
-        this._consumeEntity(TokenType.ESCAPABLE_RAW_TEXT);
-        this._beginToken(TokenType.ESCAPABLE_RAW_TEXT);
-      } else {
-        parts.push(this._readChar());
-      }
+      parts.push(this._readChar());
     }
     this._endToken([this._processCarriageReturns(parts.join(''))]);
   }
@@ -1324,7 +1341,12 @@ class _Tokenizer {
       if (this._attemptStr(INTERPOLATION.start)) {
         this._endToken([this._processCarriageReturns(parts.join(''))], current);
         parts.length = 0;
-        this._consumeInterpolation(interpolationTokenType, current, endInterpolation);
+        this._consumeInterpolation(
+          interpolationTokenType,
+          current,
+          endInterpolation,
+          textTokenType,
+        );
         this._beginToken(textTokenType);
       } else if (this._cursor.peek() === chars.$AMPERSAND) {
         this._endToken([this._processCarriageReturns(parts.join(''))]);
@@ -1352,11 +1374,13 @@ class _Tokenizer {
    * @param interpolationStart a cursor that points to the start of this interpolation.
    * @param prematureEndPredicate a function that should return true if the next characters indicate
    *     an end to the interpolation before its normal closing marker.
+   * @param textTokenType the surrounding text type, which determines whether markup is literal.
    */
   private _consumeInterpolation(
     interpolationTokenType: TokenType,
     interpolationStart: CharacterCursor,
     prematureEndPredicate: (() => boolean) | null,
+    textTokenType: TokenType,
   ): void {
     const parts: string[] = [];
     this._beginToken(interpolationTokenType, interpolationStart);
@@ -1372,7 +1396,7 @@ class _Tokenizer {
     ) {
       const current = this._cursor.clone();
 
-      if (this._isTagStart()) {
+      if (textTokenType !== TokenType.ESCAPABLE_RAW_TEXT && this._isTagStart()) {
         // We are starting what looks like an HTML element in the middle of this interpolation.
         // Reset the cursor to before the `<` character and end the interpolation token.
         // (This is actually wrong but here for backward compatibility).
@@ -1396,8 +1420,22 @@ class _Tokenizer {
       }
 
       const char = this._cursor.peek();
+      if (textTokenType === TokenType.ESCAPABLE_RAW_TEXT && char === chars.$AMPERSAND) {
+        this._readEntity();
+        continue;
+      }
       this._cursor.advance();
       if (char === chars.$BACKSLASH) {
+        if (textTokenType === TokenType.ESCAPABLE_RAW_TEXT) {
+          // String escapes cannot hide a closing tag or bypass HTML entity validation in RCDATA.
+          if (prematureEndPredicate?.()) {
+            continue;
+          }
+          if (this._cursor.peek() === chars.$AMPERSAND) {
+            this._readEntity();
+            continue;
+          }
+        }
         // Skip the next character because it was escaped.
         this._cursor.advance();
       } else if (char === inQuote) {

packages/compiler/src/ml_parser/parser.ts

@@ -437,6 +437,7 @@ class _TreeBuilder {
     while (
       this._peek.type === TokenType.INTERPOLATION ||
       this._peek.type === TokenType.TEXT ||
+      this._peek.type === TokenType.ESCAPABLE_RAW_TEXT ||
       this._peek.type === TokenType.ENCODED_ENTITY
     ) {
       token = this._advance();