From 537a59515941e6ac8941a33e59297698702c8a46 Mon Sep 17 00:00:00 2001 From: Vahid Balazadeh Date: Tue, 4 Aug 2026 22:48:56 -0400 Subject: [PATCH 1/5] Warn at fit when X contains likely-text columns High-cardinality string columns that cannot be parsed as numbers are labelled TEXT by modality detection, but nothing consumed the label: they were silently swept into the OrdinalEncoder alongside real categoricals, turning near-unique text into integer noise. fit() now emits a UserWarning naming the affected columns and pointing at the fixes (numeric dtype, tabpfn-client for genuine text, or categorical_features_indices to declare a real high-cardinality category, which also silences the warning). Co-Authored-By: Claude Fable 5 --- changelog/1159.added.md | 1 + src/tabpfn/classifier.py | 7 + src/tabpfn/regressor.py | 7 + src/tabpfn/validation.py | 60 ++++++ tests/test_classifier_interface.py | 40 ++++ .../test_preprocessing/test_data_cleaning.py | 182 +++++++++++++++++- tests/test_regressor_interface.py | 41 ++++ 7 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 changelog/1159.added.md diff --git a/changelog/1159.added.md b/changelog/1159.added.md new file mode 100644 index 000000000..f5554625d --- /dev/null +++ b/changelog/1159.added.md @@ -0,0 +1 @@ +`fit()` now warns when a column of `X` looks like free text: a string column with more than 30 distinct values that cannot be parsed as numbers. TabPFN has no text understanding, so such columns were silently encoded as near-unique integer codes, which adds noise and hurts performance. The warning names the affected columns and points to the fixes: convert the column to a numeric dtype if it holds numbers stored as strings, use the [tabpfn-client](https://github.com/PriorLabs/tabpfn-client) API if it holds genuine text, or pass the column's index in `categorical_features_indices` if it really is a high-cardinality category (which also silences the warning). diff --git a/src/tabpfn/classifier.py b/src/tabpfn/classifier.py index f034d7fca..61392c844 100644 --- a/src/tabpfn/classifier.py +++ b/src/tabpfn/classifier.py @@ -96,6 +96,7 @@ ensure_compatible_predict_input_sklearn, validate_dataset_size, validate_num_classes, + warn_if_text_features, ) if TYPE_CHECKING: @@ -727,6 +728,12 @@ def _initialize_dataset_preprocessing( max_unique_for_category=self.inference_config_.MAX_UNIQUE_FOR_CATEGORICAL_FEATURES, min_unique_for_numerical=self.inference_config_.MIN_UNIQUE_FOR_NUMERICAL_FEATURES, ) + # Must stay before `clean_data`: the TEXT labels do not survive the first + # preprocessing step that rebuilds the schema. + warn_if_text_features( + feature_schema, + declared_categorical_indices=self.categorical_features_indices, + ) X, ordinal_encoder, feature_schema = clean_data( X=X, feature_schema=feature_schema, diff --git a/src/tabpfn/regressor.py b/src/tabpfn/regressor.py index db7ec628a..e71027c51 100644 --- a/src/tabpfn/regressor.py +++ b/src/tabpfn/regressor.py @@ -89,6 +89,7 @@ ensure_compatible_fit_inputs, ensure_compatible_predict_input_sklearn, validate_dataset_size, + warn_if_text_features, ) if TYPE_CHECKING: @@ -832,6 +833,12 @@ def _initialize_dataset_preprocessing( max_unique_for_category=self.inference_config_.MAX_UNIQUE_FOR_CATEGORICAL_FEATURES, min_unique_for_numerical=self.inference_config_.MIN_UNIQUE_FOR_NUMERICAL_FEATURES, ) + # Must stay before `clean_data`: the TEXT labels do not survive the first + # preprocessing step that rebuilds the schema. + warn_if_text_features( + feature_schema, + declared_categorical_indices=self.categorical_features_indices, + ) X, ordinal_encoder, feature_schema = clean_data( X=X, feature_schema=feature_schema, diff --git a/src/tabpfn/validation.py b/src/tabpfn/validation.py index 3cd171509..fe7ee1468 100644 --- a/src/tabpfn/validation.py +++ b/src/tabpfn/validation.py @@ -21,6 +21,7 @@ from tabpfn.errors import TabPFNValidationError from tabpfn.misc._sklearn_compat import check_array, validate_data from tabpfn.preprocessing.clean import coerce_nullable_dtypes_to_numpy +from tabpfn.preprocessing.datamodel import INPUT_FEATURE_PREFIX, FeatureModality from tabpfn.settings import settings if TYPE_CHECKING: @@ -30,9 +31,14 @@ from tabpfn import TabPFNClassifier, TabPFNRegressor from tabpfn.constants import XType, YType + from tabpfn.preprocessing.datamodel import FeatureSchema T = TypeVar("T") +#: Cap on how many column names the likely-text warning lists, so a wide frame of +#: text columns does not produce an unreadable multi-kilobyte message. +_MAX_TEXT_COLUMNS_IN_WARNING = 10 + def ensure_compatible_fit_inputs( X: XType, @@ -155,6 +161,60 @@ def validate_dataset_size( ) +def warn_if_text_features( + feature_schema: FeatureSchema, + *, + declared_categorical_indices: Sequence[int] | None = None, +) -> None: + """Warn when input columns look like free text rather than categoricals. + + High-cardinality string columns are labelled `FeatureModality.TEXT` by + `detect_feature_modalities`, but this package has no text handling: they are swept + into the same `OrdinalEncoder` as real categoricals, which selects columns by dtype + (see `get_ordinal_encoder`). That turns near-unique text into near-unique integer + codes, i.e. noise rather than signal, without any error to hint at it. + + Must be called while the schema still carries the TEXT labels, i.e. before the + first preprocessing step that rebuilds it, since + `FeatureSchema.from_only_categorical_indices` collapses TEXT into NUMERICAL. + + Args: + feature_schema: The schema produced by `detect_feature_modalities`. + declared_categorical_indices: Positional indices the caller passed as + `categorical_features_indices`. These are never reported: declaring a + column categorical states that the user already knows it holds + non-numeric values and intends them as categories, so warning about it + would be noise. + """ + declared = set(declared_categorical_indices or ()) + text_names = [ + feature.name.removeprefix(INPUT_FEATURE_PREFIX) + for index, feature in enumerate(feature_schema.features) + if feature.modality is FeatureModality.TEXT and index not in declared + ] + if not text_names: + return + + shown = text_names[:_MAX_TEXT_COLUMNS_IN_WARNING] + columns = ", ".join(repr(name) for name in shown) + if len(text_names) > len(shown): + columns += f" (and {len(text_names) - len(shown)} more)" + + warnings.warn( + f"These columns look like free text and are being ordinal-encoded as " + f"high-cardinality categoricals, which usually adds noise rather than " + f"signal: {columns}.\n" + "If such a column holds numbers stored as strings, convert it to a numeric " + "dtype. If it holds genuine text, this package has no text handling -- " + "consider the tabpfn-client API, which embeds text natively: " + "https://github.com/PriorLabs/tabpfn-client \n" + "To silence this for a column that is genuinely a high-cardinality category, " + "pass its index in `categorical_features_indices`.", + UserWarning, + stacklevel=4, + ) + + def ensure_compatible_fit_inputs_sklearn( X: XType, y: YType, diff --git a/tests/test_classifier_interface.py b/tests/test_classifier_interface.py index e6639351e..e9c77a240 100644 --- a/tests/test_classifier_interface.py +++ b/tests/test_classifier_interface.py @@ -5,6 +5,7 @@ import io import itertools import os +import warnings from collections.abc import Callable from itertools import product from typing import Literal @@ -1608,3 +1609,42 @@ def mk(seed: int, n: int = 60, f: int = 5) -> tuple[np.ndarray, np.ndarray]: fitted.predict_proba_batched(X_list, y_list, X_tests) after = fitted.predict_proba(a_x[:5]) np.testing.assert_array_equal(before, after) + + +def test__fit_with_text_column__warns() -> None: + """Fitting on a DataFrame with a free-text column warns and names it. + + Which columns count as text is unit-tested in + tests/test_preprocessing/test_data_cleaning.py; here we check the estimator: + `fit` emits the warning, declaring the column in + `categorical_features_indices` silences it, and `predict` stays quiet. + """ + n = 120 + rng = np.random.default_rng(seed=42) + X = pd.DataFrame( + { + "num": rng.normal(size=n), + "review": [f"review {i}, a fairly long sentence" for i in range(n)], + } + ) + y = rng.integers(0, 2, size=n) + + model = TabPFNClassifier(n_estimators=1, device="cpu") + with pytest.warns(UserWarning, match="look like free text") as record: + model.fit(X, y) + assert "'review'" in str(record[0].message) + + # Only `fit` runs modality detection, so `predict` must not warn again. + # catch_warnings collects any warning instead of failing on unrelated ones. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + model.predict(X) + assert not [w for w in caught if "look like free text" in str(w.message)] + + model = TabPFNClassifier( + n_estimators=1, device="cpu", categorical_features_indices=[1] + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + model.fit(X, y) + assert not [w for w in caught if "look like free text" in str(w.message)] diff --git a/tests/test_preprocessing/test_data_cleaning.py b/tests/test_preprocessing/test_data_cleaning.py index bf3c7c195..fcea844b9 100644 --- a/tests/test_preprocessing/test_data_cleaning.py +++ b/tests/test_preprocessing/test_data_cleaning.py @@ -4,6 +4,8 @@ from __future__ import annotations +import warnings + import numpy as np import pandas as pd import pytest @@ -13,9 +15,19 @@ from tabpfn.errors import TabPFNValidationError from tabpfn.preprocessing import clean_data from tabpfn.preprocessing.clean import process_text_na_dataframe -from tabpfn.preprocessing.datamodel import Feature, FeatureModality, FeatureSchema +from tabpfn.preprocessing.datamodel import ( + INPUT_FEATURE_PREFIX, + Feature, + FeatureModality, + FeatureSchema, +) +from tabpfn.preprocessing.modality_detection import detect_feature_modalities from tabpfn.preprocessing.steps.preprocessing_helpers import get_ordinal_encoder -from tabpfn.validation import ensure_compatible_fit_inputs +from tabpfn.validation import ( + _MAX_TEXT_COLUMNS_IN_WARNING, + ensure_compatible_fit_inputs, + warn_if_text_features, +) @pytest.fixture @@ -724,3 +736,169 @@ def test__process_text_na_dataframe__string_against_numeric_fit_categories() -> # encode to a valid (non-negative) code; the non-numeric "abc" -> NaN -> unknown. assert out[0, 1] == -1 assert (out[1:, 1] >= 0).all() + + +def _text_schema(*names: str) -> FeatureSchema: + """Schema of TEXT features with the `input_` prefix real input names carry.""" + return FeatureSchema( + features=[ + Feature(name=f"{INPUT_FEATURE_PREFIX}{name}", modality=FeatureModality.TEXT) + for name in names + ] + ) + + +class TestWarnIfTextFeatures: + """Tests for validation.warn_if_text_features.""" + + def test__no_text_features__does_not_warn(self) -> None: + schema = _get_schema(n_numerical_features=2, n_categorical_features=2) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + warn_if_text_features(schema) + + def test__text_features__warn_with_column_names_and_remedies(self) -> None: + with pytest.warns(UserWarning, match="look like free text") as record: + warn_if_text_features(_text_schema("review")) + + message = str(record[0].message) + # Column names are shown as the user wrote them, without the input_ prefix. + assert "'review'" in message + assert INPUT_FEATURE_PREFIX not in message + # The message must state all remedies. + assert "numeric dtype" in message + assert "https://github.com/PriorLabs/tabpfn-client" in message + assert "categorical_features_indices" in message + + def test__declared_categorical_indices__are_not_reported(self) -> None: + schema = _text_schema("sku", "review") + + with pytest.warns(UserWarning, match="look like free text") as record: + warn_if_text_features(schema, declared_categorical_indices=[0]) + message = str(record[0].message) + assert "'review'" in message + assert "'sku'" not in message + + with warnings.catch_warnings(): + warnings.simplefilter("error") + warn_if_text_features(schema, declared_categorical_indices=[0, 1]) + + def test__many_text_columns__message_is_truncated(self) -> None: + n_extra = 5 + n_columns = _MAX_TEXT_COLUMNS_IN_WARNING + n_extra + schema = _text_schema(*(f"t{i}" for i in range(n_columns))) + + with pytest.warns(UserWarning, match="look like free text") as record: + warn_if_text_features(schema) + + message = str(record[0].message) + assert f"(and {n_extra} more)" in message + assert f"'t{_MAX_TEXT_COLUMNS_IN_WARNING - 1}'" in message + assert f"'t{_MAX_TEXT_COLUMNS_IN_WARNING}'" not in message + + +def _schema_for_frame( + X: pd.DataFrame, declared: list[int] | None = None +) -> FeatureSchema: + """Run the real modality detection over a frame, as `fit()` does.""" + return detect_feature_modalities( + X=X.to_numpy(dtype=object), + feature_names=list(X.columns), + provided_categorical_indices=declared, + min_samples_for_inference=100, + max_unique_for_category=30, + min_unique_for_numerical=4, + ) + + +class TestWarnIfTextFeaturesOnRealFrames: + """`detect_feature_modalities` + `warn_if_text_features` over real columns. + + Covers which columns actually reach the warning, which the schema-level tests + above cannot: they build schemas by hand. + """ + + n_rows = 200 + + def _numeric_column(self) -> np.ndarray: + return np.random.default_rng(0).normal(size=self.n_rows) + + def test__free_text_column__warns(self) -> None: + X = pd.DataFrame( + { + "num": self._numeric_column(), + "review": [f"review {i}, a fairly long sentence" for i in range(200)], + } + ) + + with pytest.warns(UserWarning, match="look like free text") as record: + warn_if_text_features(_schema_for_frame(X)) + + assert "'review'" in str(record[0].message) + + def test__ordinary_columns__do_not_warn(self) -> None: + """Neither low-cardinality strings nor fully numeric strings are TEXT. + + The former are ordinary categoricals and the latter are + detected NUMERICAL. + """ + values = np.random.default_rng(1).normal(size=200) + X = pd.DataFrame( + { + "num": self._numeric_column(), + "color": ["red", "green", "blue"] * 66 + ["red", "red"], + "as_str": [str(round(float(v), 4)) for v in values], + } + ) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + warn_if_text_features(_schema_for_frame(X)) + + def test__numeric_column_with_one_stray_token__warns(self) -> None: + """A single non-numeric token flips a whole numeric column to TEXT. + + `_is_numeric_pandas_series` requires *every* value to be coercible, so one + stray "N/A" makes the column ordinal-encoded as a near-unique categorical. + Warning here is the point of the feature: the fix is a numeric dtype. + """ + values = np.random.default_rng(2).normal(size=200) + mostly_numeric = [str(round(float(v), 4)) for v in values] + mostly_numeric[7] = "N/A" + X = pd.DataFrame( + {"num": self._numeric_column(), "mostly_numeric": mostly_numeric} + ) + + with pytest.warns(UserWarning, match="look like free text") as record: + warn_if_text_features(_schema_for_frame(X)) + + assert "'mostly_numeric'" in str(record[0].message) + + def test__declared_categorical_columns__do_not_warn(self) -> None: + """Declaring a column categorical states intent, so it must stay quiet. + + Covers both a plain string column and an explicit pandas `category` + dtype, each above the cardinality threshold. + """ + X = pd.DataFrame( + { + "num": self._numeric_column(), + "sku": [f"sku_{i % 60}" for i in range(200)], + "sku_cat": pd.Series( + [f"sku_{i % 60}" for i in range(200)], dtype="category" + ), + } + ) + declared = [1, 2] + + # The columns really are detected as TEXT; only the declaration silences + # the warning. + schema = _schema_for_frame(X, declared) + assert schema.indices_for(FeatureModality.TEXT) == declared + with pytest.warns(UserWarning, match="look like free text"): + warn_if_text_features(_schema_for_frame(X)) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + warn_if_text_features(schema, declared_categorical_indices=declared) diff --git a/tests/test_regressor_interface.py b/tests/test_regressor_interface.py index 381446daa..cdc0c6619 100644 --- a/tests/test_regressor_interface.py +++ b/tests/test_regressor_interface.py @@ -6,11 +6,13 @@ import itertools import os import typing +import warnings from collections.abc import Callable from typing import Literal from unittest import mock import numpy as np +import pandas as pd import pytest import sklearn.datasets import torch @@ -1220,3 +1222,42 @@ def test__fit_with_differentiable_input__second_call_refreshes_target_stats() -> assert not torch.allclose(reg.raw_space_bardist_.borders, bardist_borders1), ( "raw_space_bardist_ must be rebuilt to the new target scale" ) + + +def test__fit_with_text_column__warns() -> None: + """Fitting on a DataFrame with a free-text column warns and names it. + + Which columns count as text is unit-tested in + tests/test_preprocessing/test_data_cleaning.py; here we check the estimator: + `fit` emits the warning, declaring the column in + `categorical_features_indices` silences it, and `predict` stays quiet. + """ + n = 120 + rng = np.random.default_rng(seed=42) + X = pd.DataFrame( + { + "num": rng.normal(size=n), + "review": [f"review {i}, a fairly long sentence" for i in range(n)], + } + ) + y = rng.normal(size=n) + + model = TabPFNRegressor(n_estimators=1, device="cpu") + with pytest.warns(UserWarning, match="look like free text") as record: + model.fit(X, y) + assert "'review'" in str(record[0].message) + + # Only `fit` runs modality detection, so `predict` must not warn again. + # catch_warnings collects any warning instead of failing on unrelated ones. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + model.predict(X) + assert not [w for w in caught if "look like free text" in str(w.message)] + + model = TabPFNRegressor( + n_estimators=1, device="cpu", categorical_features_indices=[1] + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + model.fit(X, y) + assert not [w for w in caught if "look like free text" in str(w.message)] From 5e0503b5b0b8419359ba18085b170a435de1f499 Mon Sep 17 00:00:00 2001 From: Vahid Balazadeh Date: Tue, 4 Aug 2026 23:14:58 -0400 Subject: [PATCH 2/5] Fix stacklevel: config_context decorator adds a contextlib frame The @config_context(...) instance decorating fit() wraps it via ContextDecorator.__call__, whose wrapper is a real frame in contextlib.py, so stacklevel=4 blamed contextlib instead of the caller's fit() line. Bump to 5 and pin it with warning.filename asserts in the interface tests. Co-Authored-By: Claude Fable 5 --- src/tabpfn/validation.py | 6 +++++- tests/test_classifier_interface.py | 3 +++ tests/test_regressor_interface.py | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/tabpfn/validation.py b/src/tabpfn/validation.py index fe7ee1468..61993c1ee 100644 --- a/src/tabpfn/validation.py +++ b/src/tabpfn/validation.py @@ -211,7 +211,11 @@ def warn_if_text_features( "To silence this for a column that is genuinely a high-cardinality category, " "pass its index in `categorical_features_indices`.", UserWarning, - stacklevel=4, + # Points at a direct `estimator.fit(X, y)` call site. Five frames out: + # this function, `_initialize_dataset_preprocessing`, `fit`, and the + # contextlib wrapper added by the `@config_context(...)` decorator on + # `fit`. Pinned by the `warning.filename` asserts in the interface tests. + stacklevel=5, ) diff --git a/tests/test_classifier_interface.py b/tests/test_classifier_interface.py index e9c77a240..52df1c1a2 100644 --- a/tests/test_classifier_interface.py +++ b/tests/test_classifier_interface.py @@ -1633,6 +1633,9 @@ def test__fit_with_text_column__warns() -> None: with pytest.warns(UserWarning, match="look like free text") as record: model.fit(X, y) assert "'review'" in str(record[0].message) + # Pins the stacklevel: the warning must blame this file's `fit` call, not a + # frame inside tabpfn or the contextlib wrapper around `fit`. + assert record[0].filename == __file__ # Only `fit` runs modality detection, so `predict` must not warn again. # catch_warnings collects any warning instead of failing on unrelated ones. diff --git a/tests/test_regressor_interface.py b/tests/test_regressor_interface.py index cdc0c6619..fd63b35b2 100644 --- a/tests/test_regressor_interface.py +++ b/tests/test_regressor_interface.py @@ -1246,6 +1246,9 @@ def test__fit_with_text_column__warns() -> None: with pytest.warns(UserWarning, match="look like free text") as record: model.fit(X, y) assert "'review'" in str(record[0].message) + # Pins the stacklevel: the warning must blame this file's `fit` call, not a + # frame inside tabpfn or the contextlib wrapper around `fit`. + assert record[0].filename == __file__ # Only `fit` runs modality detection, so `predict` must not warn again. # catch_warnings collects any warning instead of failing on unrelated ones. From 0dbad7e38c4647d37ef64b355f82acd729bb0f86 Mon Sep 17 00:00:00 2001 From: Vahid Balazadeh Date: Fri, 7 Aug 2026 09:57:13 -0400 Subject: [PATCH 3/5] Move text-feature warning into detect_feature_modalities Per review feedback, the likely-text warning now fires from inside detect_feature_modalities instead of being called separately by each estimator. This gives a single call site (the one place TEXT labels are produced) and a single place to test, so the classifier/regressor tests collapse into one parametrized test under the modality-detection suite. warn_if_text_features moves from validation to modality_detection; stacklevel goes 5 -> 6 for the extra detect_feature_modalities frame. Co-Authored-By: Claude Opus 4.8 --- changelog/1159.added.md | 2 +- src/tabpfn/classifier.py | 7 - .../preprocessing/modality_detection.py | 74 +++++- src/tabpfn/regressor.py | 7 - src/tabpfn/validation.py | 64 ----- tests/test_classifier_interface.py | 43 ---- .../test_preprocessing/test_data_cleaning.py | 176 +------------- .../test_modality_detection.py | 229 +++++++++++++++++- tests/test_regressor_interface.py | 44 ---- 9 files changed, 303 insertions(+), 343 deletions(-) diff --git a/changelog/1159.added.md b/changelog/1159.added.md index f5554625d..63398da68 100644 --- a/changelog/1159.added.md +++ b/changelog/1159.added.md @@ -1 +1 @@ -`fit()` now warns when a column of `X` looks like free text: a string column with more than 30 distinct values that cannot be parsed as numbers. TabPFN has no text understanding, so such columns were silently encoded as near-unique integer codes, which adds noise and hurts performance. The warning names the affected columns and points to the fixes: convert the column to a numeric dtype if it holds numbers stored as strings, use the [tabpfn-client](https://github.com/PriorLabs/tabpfn-client) API if it holds genuine text, or pass the column's index in `categorical_features_indices` if it really is a high-cardinality category (which also silences the warning). +`fit()` now warns when a column of `X` looks like free text. diff --git a/src/tabpfn/classifier.py b/src/tabpfn/classifier.py index 61392c844..f034d7fca 100644 --- a/src/tabpfn/classifier.py +++ b/src/tabpfn/classifier.py @@ -96,7 +96,6 @@ ensure_compatible_predict_input_sklearn, validate_dataset_size, validate_num_classes, - warn_if_text_features, ) if TYPE_CHECKING: @@ -728,12 +727,6 @@ def _initialize_dataset_preprocessing( max_unique_for_category=self.inference_config_.MAX_UNIQUE_FOR_CATEGORICAL_FEATURES, min_unique_for_numerical=self.inference_config_.MIN_UNIQUE_FOR_NUMERICAL_FEATURES, ) - # Must stay before `clean_data`: the TEXT labels do not survive the first - # preprocessing step that rebuilds the schema. - warn_if_text_features( - feature_schema, - declared_categorical_indices=self.categorical_features_indices, - ) X, ordinal_encoder, feature_schema = clean_data( X=X, feature_schema=feature_schema, diff --git a/src/tabpfn/preprocessing/modality_detection.py b/src/tabpfn/preprocessing/modality_detection.py index 25b77bebd..ecbf383a6 100644 --- a/src/tabpfn/preprocessing/modality_detection.py +++ b/src/tabpfn/preprocessing/modality_detection.py @@ -4,6 +4,7 @@ from __future__ import annotations +import warnings from collections.abc import Sequence from typing import TYPE_CHECKING @@ -11,6 +12,7 @@ from tabpfn.errors import TabPFNUserError from tabpfn.preprocessing.datamodel import ( + INPUT_FEATURE_PREFIX, Feature, FeatureModality, FeatureSchema, @@ -22,6 +24,10 @@ _EARLY_EXIT_PREFIX_ROWS = 1024 +#: Cap on how many column names the likely-text warning lists, so a wide frame of +#: text columns does not produce an unreadable multi-kilobyte message. +_MAX_TEXT_COLUMNS_IN_WARNING = 10 + def detect_feature_modalities( X: np.ndarray, @@ -75,7 +81,73 @@ def detect_feature_modalities( big_enough_n_to_infer_cat=big_enough_n_to_infer_cat, ) features.append(Feature(name=feature_name, modality=feat_modality)) - return FeatureSchema(features=features) + feature_schema = FeatureSchema(features=features) + # Warn here rather than at each call site: this is the single place the TEXT + # labels are produced, and they do not survive the first preprocessing step + # that rebuilds the schema. + warn_if_text_features( + feature_schema, + declared_categorical_indices=provided_categorical_indices, + ) + return feature_schema + + +def warn_if_text_features( + feature_schema: FeatureSchema, + *, + declared_categorical_indices: Sequence[int] | None = None, +) -> None: + """Warn when input columns look like free text rather than categoricals. + + High-cardinality string columns are labelled `FeatureModality.TEXT` by + `detect_feature_modalities`, but this package has no text handling: they are swept + into the same `OrdinalEncoder` as real categoricals, which selects columns by dtype + (see `get_ordinal_encoder`). That turns near-unique text into near-unique integer + codes, i.e. noise rather than signal, without any error to hint at it. + + Called by `detect_feature_modalities` while the schema still carries the TEXT + labels, i.e. before the first preprocessing step that rebuilds it, since + `FeatureSchema.from_only_categorical_indices` collapses TEXT into NUMERICAL. + + Args: + feature_schema: The schema produced by `detect_feature_modalities`. + declared_categorical_indices: Positional indices the caller passed as + `categorical_features_indices`. These are never reported: declaring a + column categorical states that the user already knows it holds + non-numeric values and intends them as categories, so warning about it + would be noise. + """ + declared = set(declared_categorical_indices or ()) + text_names = [ + feature.name.removeprefix(INPUT_FEATURE_PREFIX) + for index, feature in enumerate(feature_schema.features) + if feature.modality is FeatureModality.TEXT and index not in declared + ] + if not text_names: + return + + shown = text_names[:_MAX_TEXT_COLUMNS_IN_WARNING] + column_names_to_print = ", ".join(repr(name) for name in shown) + if len(text_names) > len(shown): + column_names_to_print += f" (and {len(text_names) - len(shown)} more)" + + warnings.warn( + f"These columns look like free text and are being ordinal-encoded as " + f"high-cardinality categoricals, which usually adds noise rather than " + f"signal: {column_names_to_print}.\n" + "If such a column holds numbers stored as strings, convert it to a numeric " + "dtype. If it holds genuine text, this package has no text handling -- " + "consider the tabpfn-client API, which embeds text natively: " + "https://github.com/PriorLabs/tabpfn-client \n" + "To silence this for a column that is genuinely a high-cardinality category, " + "pass its index in `categorical_features_indices`.", + UserWarning, + # Points at a direct `estimator.fit(X, y)` call site. Six frames out: this + # function, `detect_feature_modalities`, `_initialize_dataset_preprocessing`, + # `fit`, and the contextlib wrapper added by the `@config_context(...)` + # decorator on `fit`. Pinned by the `warning.filename` asserts in the tests. + stacklevel=6, + ) def _detect_feature_modality( diff --git a/src/tabpfn/regressor.py b/src/tabpfn/regressor.py index e71027c51..db7ec628a 100644 --- a/src/tabpfn/regressor.py +++ b/src/tabpfn/regressor.py @@ -89,7 +89,6 @@ ensure_compatible_fit_inputs, ensure_compatible_predict_input_sklearn, validate_dataset_size, - warn_if_text_features, ) if TYPE_CHECKING: @@ -833,12 +832,6 @@ def _initialize_dataset_preprocessing( max_unique_for_category=self.inference_config_.MAX_UNIQUE_FOR_CATEGORICAL_FEATURES, min_unique_for_numerical=self.inference_config_.MIN_UNIQUE_FOR_NUMERICAL_FEATURES, ) - # Must stay before `clean_data`: the TEXT labels do not survive the first - # preprocessing step that rebuilds the schema. - warn_if_text_features( - feature_schema, - declared_categorical_indices=self.categorical_features_indices, - ) X, ordinal_encoder, feature_schema = clean_data( X=X, feature_schema=feature_schema, diff --git a/src/tabpfn/validation.py b/src/tabpfn/validation.py index 61993c1ee..3cd171509 100644 --- a/src/tabpfn/validation.py +++ b/src/tabpfn/validation.py @@ -21,7 +21,6 @@ from tabpfn.errors import TabPFNValidationError from tabpfn.misc._sklearn_compat import check_array, validate_data from tabpfn.preprocessing.clean import coerce_nullable_dtypes_to_numpy -from tabpfn.preprocessing.datamodel import INPUT_FEATURE_PREFIX, FeatureModality from tabpfn.settings import settings if TYPE_CHECKING: @@ -31,14 +30,9 @@ from tabpfn import TabPFNClassifier, TabPFNRegressor from tabpfn.constants import XType, YType - from tabpfn.preprocessing.datamodel import FeatureSchema T = TypeVar("T") -#: Cap on how many column names the likely-text warning lists, so a wide frame of -#: text columns does not produce an unreadable multi-kilobyte message. -_MAX_TEXT_COLUMNS_IN_WARNING = 10 - def ensure_compatible_fit_inputs( X: XType, @@ -161,64 +155,6 @@ def validate_dataset_size( ) -def warn_if_text_features( - feature_schema: FeatureSchema, - *, - declared_categorical_indices: Sequence[int] | None = None, -) -> None: - """Warn when input columns look like free text rather than categoricals. - - High-cardinality string columns are labelled `FeatureModality.TEXT` by - `detect_feature_modalities`, but this package has no text handling: they are swept - into the same `OrdinalEncoder` as real categoricals, which selects columns by dtype - (see `get_ordinal_encoder`). That turns near-unique text into near-unique integer - codes, i.e. noise rather than signal, without any error to hint at it. - - Must be called while the schema still carries the TEXT labels, i.e. before the - first preprocessing step that rebuilds it, since - `FeatureSchema.from_only_categorical_indices` collapses TEXT into NUMERICAL. - - Args: - feature_schema: The schema produced by `detect_feature_modalities`. - declared_categorical_indices: Positional indices the caller passed as - `categorical_features_indices`. These are never reported: declaring a - column categorical states that the user already knows it holds - non-numeric values and intends them as categories, so warning about it - would be noise. - """ - declared = set(declared_categorical_indices or ()) - text_names = [ - feature.name.removeprefix(INPUT_FEATURE_PREFIX) - for index, feature in enumerate(feature_schema.features) - if feature.modality is FeatureModality.TEXT and index not in declared - ] - if not text_names: - return - - shown = text_names[:_MAX_TEXT_COLUMNS_IN_WARNING] - columns = ", ".join(repr(name) for name in shown) - if len(text_names) > len(shown): - columns += f" (and {len(text_names) - len(shown)} more)" - - warnings.warn( - f"These columns look like free text and are being ordinal-encoded as " - f"high-cardinality categoricals, which usually adds noise rather than " - f"signal: {columns}.\n" - "If such a column holds numbers stored as strings, convert it to a numeric " - "dtype. If it holds genuine text, this package has no text handling -- " - "consider the tabpfn-client API, which embeds text natively: " - "https://github.com/PriorLabs/tabpfn-client \n" - "To silence this for a column that is genuinely a high-cardinality category, " - "pass its index in `categorical_features_indices`.", - UserWarning, - # Points at a direct `estimator.fit(X, y)` call site. Five frames out: - # this function, `_initialize_dataset_preprocessing`, `fit`, and the - # contextlib wrapper added by the `@config_context(...)` decorator on - # `fit`. Pinned by the `warning.filename` asserts in the interface tests. - stacklevel=5, - ) - - def ensure_compatible_fit_inputs_sklearn( X: XType, y: YType, diff --git a/tests/test_classifier_interface.py b/tests/test_classifier_interface.py index 52df1c1a2..e6639351e 100644 --- a/tests/test_classifier_interface.py +++ b/tests/test_classifier_interface.py @@ -5,7 +5,6 @@ import io import itertools import os -import warnings from collections.abc import Callable from itertools import product from typing import Literal @@ -1609,45 +1608,3 @@ def mk(seed: int, n: int = 60, f: int = 5) -> tuple[np.ndarray, np.ndarray]: fitted.predict_proba_batched(X_list, y_list, X_tests) after = fitted.predict_proba(a_x[:5]) np.testing.assert_array_equal(before, after) - - -def test__fit_with_text_column__warns() -> None: - """Fitting on a DataFrame with a free-text column warns and names it. - - Which columns count as text is unit-tested in - tests/test_preprocessing/test_data_cleaning.py; here we check the estimator: - `fit` emits the warning, declaring the column in - `categorical_features_indices` silences it, and `predict` stays quiet. - """ - n = 120 - rng = np.random.default_rng(seed=42) - X = pd.DataFrame( - { - "num": rng.normal(size=n), - "review": [f"review {i}, a fairly long sentence" for i in range(n)], - } - ) - y = rng.integers(0, 2, size=n) - - model = TabPFNClassifier(n_estimators=1, device="cpu") - with pytest.warns(UserWarning, match="look like free text") as record: - model.fit(X, y) - assert "'review'" in str(record[0].message) - # Pins the stacklevel: the warning must blame this file's `fit` call, not a - # frame inside tabpfn or the contextlib wrapper around `fit`. - assert record[0].filename == __file__ - - # Only `fit` runs modality detection, so `predict` must not warn again. - # catch_warnings collects any warning instead of failing on unrelated ones. - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - model.predict(X) - assert not [w for w in caught if "look like free text" in str(w.message)] - - model = TabPFNClassifier( - n_estimators=1, device="cpu", categorical_features_indices=[1] - ) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - model.fit(X, y) - assert not [w for w in caught if "look like free text" in str(w.message)] diff --git a/tests/test_preprocessing/test_data_cleaning.py b/tests/test_preprocessing/test_data_cleaning.py index fcea844b9..ee778472f 100644 --- a/tests/test_preprocessing/test_data_cleaning.py +++ b/tests/test_preprocessing/test_data_cleaning.py @@ -4,8 +4,6 @@ from __future__ import annotations -import warnings - import numpy as np import pandas as pd import pytest @@ -16,18 +14,12 @@ from tabpfn.preprocessing import clean_data from tabpfn.preprocessing.clean import process_text_na_dataframe from tabpfn.preprocessing.datamodel import ( - INPUT_FEATURE_PREFIX, Feature, FeatureModality, FeatureSchema, ) -from tabpfn.preprocessing.modality_detection import detect_feature_modalities from tabpfn.preprocessing.steps.preprocessing_helpers import get_ordinal_encoder -from tabpfn.validation import ( - _MAX_TEXT_COLUMNS_IN_WARNING, - ensure_compatible_fit_inputs, - warn_if_text_features, -) +from tabpfn.validation import ensure_compatible_fit_inputs @pytest.fixture @@ -736,169 +728,3 @@ def test__process_text_na_dataframe__string_against_numeric_fit_categories() -> # encode to a valid (non-negative) code; the non-numeric "abc" -> NaN -> unknown. assert out[0, 1] == -1 assert (out[1:, 1] >= 0).all() - - -def _text_schema(*names: str) -> FeatureSchema: - """Schema of TEXT features with the `input_` prefix real input names carry.""" - return FeatureSchema( - features=[ - Feature(name=f"{INPUT_FEATURE_PREFIX}{name}", modality=FeatureModality.TEXT) - for name in names - ] - ) - - -class TestWarnIfTextFeatures: - """Tests for validation.warn_if_text_features.""" - - def test__no_text_features__does_not_warn(self) -> None: - schema = _get_schema(n_numerical_features=2, n_categorical_features=2) - - with warnings.catch_warnings(): - warnings.simplefilter("error") - warn_if_text_features(schema) - - def test__text_features__warn_with_column_names_and_remedies(self) -> None: - with pytest.warns(UserWarning, match="look like free text") as record: - warn_if_text_features(_text_schema("review")) - - message = str(record[0].message) - # Column names are shown as the user wrote them, without the input_ prefix. - assert "'review'" in message - assert INPUT_FEATURE_PREFIX not in message - # The message must state all remedies. - assert "numeric dtype" in message - assert "https://github.com/PriorLabs/tabpfn-client" in message - assert "categorical_features_indices" in message - - def test__declared_categorical_indices__are_not_reported(self) -> None: - schema = _text_schema("sku", "review") - - with pytest.warns(UserWarning, match="look like free text") as record: - warn_if_text_features(schema, declared_categorical_indices=[0]) - message = str(record[0].message) - assert "'review'" in message - assert "'sku'" not in message - - with warnings.catch_warnings(): - warnings.simplefilter("error") - warn_if_text_features(schema, declared_categorical_indices=[0, 1]) - - def test__many_text_columns__message_is_truncated(self) -> None: - n_extra = 5 - n_columns = _MAX_TEXT_COLUMNS_IN_WARNING + n_extra - schema = _text_schema(*(f"t{i}" for i in range(n_columns))) - - with pytest.warns(UserWarning, match="look like free text") as record: - warn_if_text_features(schema) - - message = str(record[0].message) - assert f"(and {n_extra} more)" in message - assert f"'t{_MAX_TEXT_COLUMNS_IN_WARNING - 1}'" in message - assert f"'t{_MAX_TEXT_COLUMNS_IN_WARNING}'" not in message - - -def _schema_for_frame( - X: pd.DataFrame, declared: list[int] | None = None -) -> FeatureSchema: - """Run the real modality detection over a frame, as `fit()` does.""" - return detect_feature_modalities( - X=X.to_numpy(dtype=object), - feature_names=list(X.columns), - provided_categorical_indices=declared, - min_samples_for_inference=100, - max_unique_for_category=30, - min_unique_for_numerical=4, - ) - - -class TestWarnIfTextFeaturesOnRealFrames: - """`detect_feature_modalities` + `warn_if_text_features` over real columns. - - Covers which columns actually reach the warning, which the schema-level tests - above cannot: they build schemas by hand. - """ - - n_rows = 200 - - def _numeric_column(self) -> np.ndarray: - return np.random.default_rng(0).normal(size=self.n_rows) - - def test__free_text_column__warns(self) -> None: - X = pd.DataFrame( - { - "num": self._numeric_column(), - "review": [f"review {i}, a fairly long sentence" for i in range(200)], - } - ) - - with pytest.warns(UserWarning, match="look like free text") as record: - warn_if_text_features(_schema_for_frame(X)) - - assert "'review'" in str(record[0].message) - - def test__ordinary_columns__do_not_warn(self) -> None: - """Neither low-cardinality strings nor fully numeric strings are TEXT. - - The former are ordinary categoricals and the latter are - detected NUMERICAL. - """ - values = np.random.default_rng(1).normal(size=200) - X = pd.DataFrame( - { - "num": self._numeric_column(), - "color": ["red", "green", "blue"] * 66 + ["red", "red"], - "as_str": [str(round(float(v), 4)) for v in values], - } - ) - - with warnings.catch_warnings(): - warnings.simplefilter("error") - warn_if_text_features(_schema_for_frame(X)) - - def test__numeric_column_with_one_stray_token__warns(self) -> None: - """A single non-numeric token flips a whole numeric column to TEXT. - - `_is_numeric_pandas_series` requires *every* value to be coercible, so one - stray "N/A" makes the column ordinal-encoded as a near-unique categorical. - Warning here is the point of the feature: the fix is a numeric dtype. - """ - values = np.random.default_rng(2).normal(size=200) - mostly_numeric = [str(round(float(v), 4)) for v in values] - mostly_numeric[7] = "N/A" - X = pd.DataFrame( - {"num": self._numeric_column(), "mostly_numeric": mostly_numeric} - ) - - with pytest.warns(UserWarning, match="look like free text") as record: - warn_if_text_features(_schema_for_frame(X)) - - assert "'mostly_numeric'" in str(record[0].message) - - def test__declared_categorical_columns__do_not_warn(self) -> None: - """Declaring a column categorical states intent, so it must stay quiet. - - Covers both a plain string column and an explicit pandas `category` - dtype, each above the cardinality threshold. - """ - X = pd.DataFrame( - { - "num": self._numeric_column(), - "sku": [f"sku_{i % 60}" for i in range(200)], - "sku_cat": pd.Series( - [f"sku_{i % 60}" for i in range(200)], dtype="category" - ), - } - ) - declared = [1, 2] - - # The columns really are detected as TEXT; only the declaration silences - # the warning. - schema = _schema_for_frame(X, declared) - assert schema.indices_for(FeatureModality.TEXT) == declared - with pytest.warns(UserWarning, match="look like free text"): - warn_if_text_features(_schema_for_frame(X)) - - with warnings.catch_warnings(): - warnings.simplefilter("error") - warn_if_text_features(schema, declared_categorical_indices=declared) diff --git a/tests/test_preprocessing/test_modality_detection.py b/tests/test_preprocessing/test_modality_detection.py index 3989981c9..c34b0cabc 100644 --- a/tests/test_preprocessing/test_modality_detection.py +++ b/tests/test_preprocessing/test_modality_detection.py @@ -4,17 +4,26 @@ from __future__ import annotations +import warnings from typing import Any import numpy as np import pandas as pd import pytest -from tabpfn.preprocessing.datamodel import FeatureModality +from tabpfn import TabPFNClassifier, TabPFNRegressor +from tabpfn.preprocessing.datamodel import ( + INPUT_FEATURE_PREFIX, + Feature, + FeatureModality, + FeatureSchema, +) from tabpfn.preprocessing.modality_detection import ( _EARLY_EXIT_PREFIX_ROWS, + _MAX_TEXT_COLUMNS_IN_WARNING, _detect_feature_modality, detect_feature_modalities, + warn_if_text_features, ) from tabpfn.preprocessing.type_detection import infer_categorical_features @@ -427,3 +436,221 @@ def test__early_exit_not_fooled_by_uninformative_prefix(): np.concatenate([np.zeros(_EARLY_EXIT_PREFIX_ROWS), np.arange(1.0, 4000.0)]) ) assert _for_test_detect_with_defaults(s) == FeatureModality.NUMERICAL + + +def _text_schema(*names: str) -> FeatureSchema: + """Schema of TEXT features with the `input_` prefix real input names carry.""" + return FeatureSchema( + features=[ + Feature(name=f"{INPUT_FEATURE_PREFIX}{name}", modality=FeatureModality.TEXT) + for name in names + ] + ) + + +class TestWarnIfTextFeatures: + """Schema-level unit tests for `warn_if_text_features`.""" + + def test__no_text_features__does_not_warn(self) -> None: + schema = FeatureSchema( + features=[ + Feature(name="input_a", modality=FeatureModality.NUMERICAL), + Feature(name="input_b", modality=FeatureModality.CATEGORICAL), + ] + ) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + warn_if_text_features(schema) + + def test__text_features__warn_with_column_names_and_remedies(self) -> None: + with pytest.warns(UserWarning, match="look like free text") as record: + warn_if_text_features(_text_schema("review")) + + message = str(record[0].message) + # Column names are shown as the user wrote them, without the input_ prefix. + assert "'review'" in message + assert INPUT_FEATURE_PREFIX not in message + # The message must state all remedies. + assert "numeric dtype" in message + assert "https://github.com/PriorLabs/tabpfn-client" in message + assert "categorical_features_indices" in message + + def test__declared_categorical_indices__are_not_reported(self) -> None: + schema = _text_schema("sku", "review") + + with pytest.warns(UserWarning, match="look like free text") as record: + warn_if_text_features(schema, declared_categorical_indices=[0]) + message = str(record[0].message) + assert "'review'" in message + assert "'sku'" not in message + + with warnings.catch_warnings(): + warnings.simplefilter("error") + warn_if_text_features(schema, declared_categorical_indices=[0, 1]) + + def test__many_text_columns__message_is_truncated(self) -> None: + n_extra = 5 + n_columns = _MAX_TEXT_COLUMNS_IN_WARNING + n_extra + schema = _text_schema(*(f"t{i}" for i in range(n_columns))) + + with pytest.warns(UserWarning, match="look like free text") as record: + warn_if_text_features(schema) + + message = str(record[0].message) + assert f"(and {n_extra} more)" in message + assert f"'t{_MAX_TEXT_COLUMNS_IN_WARNING - 1}'" in message + assert f"'t{_MAX_TEXT_COLUMNS_IN_WARNING}'" not in message + + +class TestDetectFeatureModalitiesWarnsOnText: + """`detect_feature_modalities` emits the text warning over real columns. + + The warning is now produced inside `detect_feature_modalities`, so these + exercise the whole path: which columns actually get labelled TEXT and thus + reach the warning, which the schema-level tests above cannot (they build + schemas by hand). + """ + + n_rows = 200 + + def _numeric_column(self) -> np.ndarray: + return np.random.default_rng(0).normal(size=self.n_rows) + + def _detect( + self, X: pd.DataFrame, declared: list[int] | None = None + ) -> FeatureSchema: + """Run modality detection over a frame, as `fit()` does.""" + return detect_feature_modalities( + X=X.to_numpy(dtype=object), + feature_names=list(X.columns), + provided_categorical_indices=declared, + min_samples_for_inference=100, + max_unique_for_category=30, + min_unique_for_numerical=4, + ) + + def test__free_text_column__warns(self) -> None: + X = pd.DataFrame( + { + "num": self._numeric_column(), + "review": [f"review {i}, a fairly long sentence" for i in range(200)], + } + ) + + with pytest.warns(UserWarning, match="look like free text") as record: + self._detect(X) + + assert "'review'" in str(record[0].message) + + def test__ordinary_columns__do_not_warn(self) -> None: + """Neither low-cardinality strings nor fully numeric strings are TEXT. + + The former are ordinary categoricals and the latter are + detected NUMERICAL. + """ + values = np.random.default_rng(1).normal(size=200) + X = pd.DataFrame( + { + "num": self._numeric_column(), + "color": ["red", "green", "blue"] * 66 + ["red", "red"], + "as_str": [str(round(float(v), 4)) for v in values], + } + ) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + self._detect(X) + + def test__numeric_column_with_one_stray_token__warns(self) -> None: + """A single non-numeric token flips a whole numeric column to TEXT. + + `_is_numeric_pandas_series` requires *every* value to be coercible, so one + stray "N/A" makes the column ordinal-encoded as a near-unique categorical. + Warning here is the point of the feature: the fix is a numeric dtype. + """ + values = np.random.default_rng(2).normal(size=200) + mostly_numeric = [str(round(float(v), 4)) for v in values] + mostly_numeric[7] = "N/A" + X = pd.DataFrame( + {"num": self._numeric_column(), "mostly_numeric": mostly_numeric} + ) + + with pytest.warns(UserWarning, match="look like free text") as record: + self._detect(X) + + assert "'mostly_numeric'" in str(record[0].message) + + def test__declared_categorical_columns__do_not_warn(self) -> None: + """Declaring a column categorical states intent, so it must stay quiet. + + Covers both a plain string column and an explicit pandas `category` + dtype, each above the cardinality threshold. + """ + X = pd.DataFrame( + { + "num": self._numeric_column(), + "sku": [f"sku_{i % 60}" for i in range(200)], + "sku_cat": pd.Series( + [f"sku_{i % 60}" for i in range(200)], dtype="category" + ), + } + ) + declared = [1, 2] + + # Without the declaration the columns really are detected as TEXT and warn. + with pytest.warns(UserWarning, match="look like free text"): + self._detect(X) + + # Declaring them silences the warning; the columns are still labelled TEXT. + with warnings.catch_warnings(): + warnings.simplefilter("error") + schema = self._detect(X, declared) + assert schema.indices_for(FeatureModality.TEXT) == declared + + +@pytest.mark.parametrize("estimator_cls", [TabPFNClassifier, TabPFNRegressor]) +def test__fit_with_text_column__warns_at_call_site(estimator_cls: type) -> None: + """`fit` runs `detect_feature_modalities`, so a free-text column warns. + + Both estimators share the detection path, so one parametrized test pins the + estimator-level behaviour: `fit` emits the warning naming the column and + blaming this file's `fit` call (the stacklevel), declaring the column in + `categorical_features_indices` silences it, and `predict` stays quiet. + """ + n = 120 + rng = np.random.default_rng(seed=42) + X = pd.DataFrame( + { + "num": rng.normal(size=n), + "review": [f"review {i}, a fairly long sentence" for i in range(n)], + } + ) + y = ( + rng.integers(0, 2, size=n) + if estimator_cls is TabPFNClassifier + else rng.normal(size=n) + ) + + model = estimator_cls(n_estimators=1, device="cpu") + with pytest.warns(UserWarning, match="look like free text") as record: + model.fit(X, y) + assert "'review'" in str(record[0].message) + # Pins the stacklevel: the warning must blame this file's `fit` call, not a + # frame inside tabpfn or the contextlib wrapper around `fit`. + assert record[0].filename == __file__ + + # Only `fit` runs modality detection, so `predict` must not warn again. + # catch_warnings collects any warning instead of failing on unrelated ones. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + model.predict(X) + assert not [w for w in caught if "look like free text" in str(w.message)] + + model = estimator_cls( + n_estimators=1, device="cpu", categorical_features_indices=[1] + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + model.fit(X, y) + assert not [w for w in caught if "look like free text" in str(w.message)] diff --git a/tests/test_regressor_interface.py b/tests/test_regressor_interface.py index fd63b35b2..381446daa 100644 --- a/tests/test_regressor_interface.py +++ b/tests/test_regressor_interface.py @@ -6,13 +6,11 @@ import itertools import os import typing -import warnings from collections.abc import Callable from typing import Literal from unittest import mock import numpy as np -import pandas as pd import pytest import sklearn.datasets import torch @@ -1222,45 +1220,3 @@ def test__fit_with_differentiable_input__second_call_refreshes_target_stats() -> assert not torch.allclose(reg.raw_space_bardist_.borders, bardist_borders1), ( "raw_space_bardist_ must be rebuilt to the new target scale" ) - - -def test__fit_with_text_column__warns() -> None: - """Fitting on a DataFrame with a free-text column warns and names it. - - Which columns count as text is unit-tested in - tests/test_preprocessing/test_data_cleaning.py; here we check the estimator: - `fit` emits the warning, declaring the column in - `categorical_features_indices` silences it, and `predict` stays quiet. - """ - n = 120 - rng = np.random.default_rng(seed=42) - X = pd.DataFrame( - { - "num": rng.normal(size=n), - "review": [f"review {i}, a fairly long sentence" for i in range(n)], - } - ) - y = rng.normal(size=n) - - model = TabPFNRegressor(n_estimators=1, device="cpu") - with pytest.warns(UserWarning, match="look like free text") as record: - model.fit(X, y) - assert "'review'" in str(record[0].message) - # Pins the stacklevel: the warning must blame this file's `fit` call, not a - # frame inside tabpfn or the contextlib wrapper around `fit`. - assert record[0].filename == __file__ - - # Only `fit` runs modality detection, so `predict` must not warn again. - # catch_warnings collects any warning instead of failing on unrelated ones. - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - model.predict(X) - assert not [w for w in caught if "look like free text" in str(w.message)] - - model = TabPFNRegressor( - n_estimators=1, device="cpu", categorical_features_indices=[1] - ) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - model.fit(X, y) - assert not [w for w in caught if "look like free text" in str(w.message)] From 067e9f9ef24b929d97853b8c0c5560f44c90193f Mon Sep 17 00:00:00 2001 From: Vahid Balazadeh Date: Fri, 7 Aug 2026 10:37:41 -0400 Subject: [PATCH 4/5] Fix import statements in test_data_cleaning.py Refactor import statements for cleaner code. --- tests/test_preprocessing/test_data_cleaning.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_preprocessing/test_data_cleaning.py b/tests/test_preprocessing/test_data_cleaning.py index ee778472f..bf3c7c195 100644 --- a/tests/test_preprocessing/test_data_cleaning.py +++ b/tests/test_preprocessing/test_data_cleaning.py @@ -13,11 +13,7 @@ from tabpfn.errors import TabPFNValidationError from tabpfn.preprocessing import clean_data from tabpfn.preprocessing.clean import process_text_na_dataframe -from tabpfn.preprocessing.datamodel import ( - Feature, - FeatureModality, - FeatureSchema, -) +from tabpfn.preprocessing.datamodel import Feature, FeatureModality, FeatureSchema from tabpfn.preprocessing.steps.preprocessing_helpers import get_ordinal_encoder from tabpfn.validation import ensure_compatible_fit_inputs From 176071d61f6e5906873b32fd7c9a10e95739fcac Mon Sep 17 00:00:00 2001 From: Vahid Balazadeh Date: Mon, 10 Aug 2026 06:56:19 -0400 Subject: [PATCH 5/5] Make the warning method private and remove unnecessary comments --- src/tabpfn/preprocessing/modality_detection.py | 7 ++----- tests/test_preprocessing/test_modality_detection.py | 12 ++++++------ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/tabpfn/preprocessing/modality_detection.py b/src/tabpfn/preprocessing/modality_detection.py index ecbf383a6..ab7eb7c8c 100644 --- a/src/tabpfn/preprocessing/modality_detection.py +++ b/src/tabpfn/preprocessing/modality_detection.py @@ -82,17 +82,14 @@ def detect_feature_modalities( ) features.append(Feature(name=feature_name, modality=feat_modality)) feature_schema = FeatureSchema(features=features) - # Warn here rather than at each call site: this is the single place the TEXT - # labels are produced, and they do not survive the first preprocessing step - # that rebuilds the schema. - warn_if_text_features( + _warn_if_text_features( feature_schema, declared_categorical_indices=provided_categorical_indices, ) return feature_schema -def warn_if_text_features( +def _warn_if_text_features( feature_schema: FeatureSchema, *, declared_categorical_indices: Sequence[int] | None = None, diff --git a/tests/test_preprocessing/test_modality_detection.py b/tests/test_preprocessing/test_modality_detection.py index c34b0cabc..80b00b48f 100644 --- a/tests/test_preprocessing/test_modality_detection.py +++ b/tests/test_preprocessing/test_modality_detection.py @@ -22,8 +22,8 @@ _EARLY_EXIT_PREFIX_ROWS, _MAX_TEXT_COLUMNS_IN_WARNING, _detect_feature_modality, + _warn_if_text_features, detect_feature_modalities, - warn_if_text_features, ) from tabpfn.preprocessing.type_detection import infer_categorical_features @@ -461,11 +461,11 @@ def test__no_text_features__does_not_warn(self) -> None: with warnings.catch_warnings(): warnings.simplefilter("error") - warn_if_text_features(schema) + _warn_if_text_features(schema) def test__text_features__warn_with_column_names_and_remedies(self) -> None: with pytest.warns(UserWarning, match="look like free text") as record: - warn_if_text_features(_text_schema("review")) + _warn_if_text_features(_text_schema("review")) message = str(record[0].message) # Column names are shown as the user wrote them, without the input_ prefix. @@ -480,14 +480,14 @@ def test__declared_categorical_indices__are_not_reported(self) -> None: schema = _text_schema("sku", "review") with pytest.warns(UserWarning, match="look like free text") as record: - warn_if_text_features(schema, declared_categorical_indices=[0]) + _warn_if_text_features(schema, declared_categorical_indices=[0]) message = str(record[0].message) assert "'review'" in message assert "'sku'" not in message with warnings.catch_warnings(): warnings.simplefilter("error") - warn_if_text_features(schema, declared_categorical_indices=[0, 1]) + _warn_if_text_features(schema, declared_categorical_indices=[0, 1]) def test__many_text_columns__message_is_truncated(self) -> None: n_extra = 5 @@ -495,7 +495,7 @@ def test__many_text_columns__message_is_truncated(self) -> None: schema = _text_schema(*(f"t{i}" for i in range(n_columns))) with pytest.warns(UserWarning, match="look like free text") as record: - warn_if_text_features(schema) + _warn_if_text_features(schema) message = str(record[0].message) assert f"(and {n_extra} more)" in message