gh-153837: Fix IndentationError offset in tokenize to point to the indentation error, not end of line

Author: Alessio Attilio

Lib/test/test_tokenize.py

@@ -151,9 +151,26 @@ def k(x):
         self.assertEqual(
             e.exception.msg,
             'unindent does not match any outer indentation level')
-        self.assertEqual(e.exception.offset, 9)
+        self.assertEqual(e.exception.offset, 2)
         self.assertEqual(e.exception.text, '  x += 5')
 
+        # gh-153837: offset should point to the indentation error, not end of line
+        indent_error_long_line = b"""\
+def foo():
+    x = 1
+   x = "some_very_long_expression"
+"""
+        readline = BytesIO(indent_error_long_line).readline
+        with self.assertRaisesRegex(IndentationError,
+                                    "unindent does not match any "
+                                    "outer indentation level") as e:
+            for tok in tokenize.tokenize(readline):
+                pass
+        self.assertEqual(e.exception.lineno, 3)
+        self.assertEqual(e.exception.offset, 3)
+        self.assertEqual(e.exception.text,
+                         '   x = "some_very_long_expression"')
+
     def test_int(self):
         # Ordinary integers and binary operators
         self.check_tokenize("0xff <= 255", """\

Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-19-55-00.gh-issue-153837.bXYZ12.rst

@@ -0,0 +1 @@
+Fix incorrect column offset in :exc:`IndentationError` raised by the :mod:`tokenize` module. The offset now points to the first non-whitespace character on the line where the indentation error occurs, instead of incorrectly pointing to the end of the line.

Python/Python-tokenize.c

@@ -148,7 +148,15 @@ _tokenizer_error(tokenizeriterobject *it)
         goto exit;
     }
 
-    Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, tok->inp - tok->buf);
+    Py_ssize_t byte_offset = tok->inp - tok->buf;
+    if (tok->done == E_DEDENT || tok->done == E_TABSPACE || tok->done == E_TOODEEP) {
+        const char *line_start = tok->line_start ? tok->line_start : tok->buf;
+        byte_offset = line_start - tok->buf;
+        while (byte_offset < size && (tok->buf[byte_offset] == ' ' || tok->buf[byte_offset] == '\t')) {
+            byte_offset++;
+        }
+    }
+    Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, byte_offset);
     if (offset == -1) {
         result = -1;
         goto exit;