oversampleqa

oversampleqa

oversampleqa: A diagnostic toolkit for validating oversampling methods.

This package implements validation methods for synthetic data generated by oversampling techniques like SMOTE, ADASYN, and their variants.

SCHEMA_VERSION = '1.0' module-attribute

Version of the exported JSON structure.

Bump the minor part for additive changes and the major part when a field is removed or changes meaning. Consumers should refuse a major version they do not recognise rather than guess.

ReferenceSet = Literal['hidden_minority', 'train_minority'] module-attribute

Which minority set validation compares synthetic points against.

DatasetRepository

Repository for curated real-world and synthetic benchmarking datasets.

Source code in src/oversampleqa/advanced_benchmark.py

class DatasetRepository:
    """Repository for curated real-world and synthetic benchmarking datasets."""

    def __init__(self, cache_dir: str = ".oversampleqa_datasets") -> None:
        self.cache_dir = Path(cache_dir)
        self.cache_dir.mkdir(parents=True, exist_ok=True)

    def load_research_datasets(
        self,
        domains: Sequence[str] | None = None,
        max_samples: int = 10_000,
        include_openml: bool = False,
    ) -> list[dict[str, Any]]:
        """Load curated datasets for benchmarking.

        Args:
            domains: Domain names to load.
            max_samples: Maximum number of samples per dataset.
            include_openml: Whether to attempt OpenML downloads.

        Returns:
            List of dataset descriptors.
        """
        domains = tuple(domains or ("medical", "financial"))
        datasets: list[dict[str, Any]] = []
        for domain in domains:
            datasets.extend(self._load_domain(domain, max_samples, include_openml))
        return datasets

    def _load_domain(
        self, domain: str, max_samples: int, include_openml: bool
    ) -> list[dict[str, Any]]:
        """Load datasets for a single domain.

        Args:
            domain: Domain name.
            max_samples: Maximum number of samples per dataset.
            include_openml: Whether to attempt OpenML downloads.

        Returns:
            List of dataset descriptors.
        """
        domain = domain.lower()
        if domain == "medical":
            return self._load_medical(max_samples, include_openml)
        if domain == "financial":
            return self._load_financial(max_samples)
        return []

    def _load_medical(
        self, max_samples: int, include_openml: bool
    ) -> list[dict[str, Any]]:
        """Load medical datasets for benchmarking.

        Args:
            max_samples: Maximum number of samples per dataset.
            include_openml: Whether to attempt OpenML downloads.

        Returns:
            List of dataset descriptors.
        """
        from sklearn.datasets import load_breast_cancer

        cancer = load_breast_cancer()
        X, y = cancer.data[:max_samples], cancer.target[:max_samples]
        datasets = [
            {
                "name": "breast_cancer",
                "data": X,
                "target": y,
                # Derived, not declared. The full dataset's minority is class 0
                # (212 malignant against 357 benign) and this said 1, so the
                # benchmark oversampled the majority. max_samples then inverts
                # it again -- the first 200 rows are 104 class-0 to 96 class-1.
                "minority_label": infer_minority_label(y),
                "provenance": bundled_provenance(
                    "sklearn.datasets.load_breast_cancer",
                    url="https://archive.ics.uci.edu/dataset/17/breast+cancer+wisconsin+diagnostic",
                    license="CC BY 4.0 (UCI ML Repository); redistributed with scikit-learn.",
                    notes=(
                        f"Truncated to the first {max_samples} rows, which is a "
                        "positional slice and not a random sample."
                    ),
                    max_samples=max_samples,
                ),
            }
        ]
        if include_openml:
            try:
                from sklearn.datasets import fetch_openml

                diabetes = fetch_openml("diabetes", version=1, as_frame=False)
                Xd = diabetes.data[:max_samples]
                yd = (diabetes.target[:max_samples] == "tested_positive").astype(int)
                datasets.append(
                    {
                        "name": "diabetes_openml",
                        "data": Xd,
                        "target": yd,
                        "minority_label": infer_minority_label(yd),
                        "provenance": openml_provenance(
                            "diabetes",
                            1,
                            notes=(
                                "Version pinned to 1. Target binarised as "
                                "'tested_positive'. Truncated to the first "
                                f"{max_samples} rows."
                            ),
                        ),
                    }
                )
            except Exception as exc:  # pragma: no cover - network dependent
                warnings.warn(f"OpenML download failed: {exc}", stacklevel=2)
        return datasets

    def _load_financial(self, max_samples: int) -> list[dict[str, Any]]:
        """Load financial datasets for benchmarking.

        Args:
            max_samples: Maximum number of samples per dataset.

        Returns:
            List of dataset descriptors.
        """
        try:
            from imbalanced_datasets import creditcard  # optional
        except Exception:  # pragma: no cover
            creditcard = None

        datasets: list[dict[str, Any]] = []
        if creditcard is not None:  # pragma: no cover - optional dependency
            X, y = creditcard.load_data()
            datasets.append(
                {
                    "name": "creditcard",
                    "data": X[:max_samples],
                    "target": y[:max_samples],
                    "minority_label": infer_minority_label(y[:max_samples]),
                    "provenance": {
                        "source": "third-party",
                        "generator": "imbalanced_datasets.creditcard.load_data",
                        "params": {"max_samples": max_samples},
                        "url": "https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud",
                        "license": (
                            "Database Contents License (DbCL) v1.0. Redistribution "
                            "terms are the upstream package's, not this project's."
                        ),
                        "notes": (
                            "Optional dependency; the loader pins no version, so "
                            "the contents are whatever the installed release "
                            f"ships. Truncated to the first {max_samples} rows."
                        ),
                    },
                }
            )
        return datasets

    def create_synthetic_benchmark_suite(
        self, difficulty_levels: Sequence[str] | None = None
    ) -> list[dict[str, Any]]:
        """Generate synthetic datasets for the requested difficulty levels.

        Args:
            difficulty_levels: Difficulty labels to generate.

        Returns:
            List of synthetic dataset descriptors.
        """
        difficulty_levels = tuple(
            difficulty_levels or ("easy", "medium", "hard", "extreme")
        )
        synthetic: list[dict[str, Any]] = []
        for difficulty in difficulty_levels:
            synthetic.extend(self._generate_difficulty(difficulty))
        return synthetic

    def _generate_difficulty(self, difficulty: str) -> list[dict[str, Any]]:
        """Create synthetic datasets for a single difficulty tier.

        Args:
            difficulty: Difficulty label.

        Returns:
            List of dataset descriptors for the difficulty tier.
        """
        from sklearn.datasets import make_classification

        difficulty = difficulty.lower()
        configs: list[dict[str, Any]]
        if difficulty == "easy":
            configs = [
                {
                    "n_samples": 600,
                    "n_features": 8,
                    "class_sep": 2.0,
                    "weights": [0.75, 0.25],
                },
                {
                    "n_samples": 800,
                    "n_features": 5,
                    "class_sep": 1.8,
                    "weights": [0.8, 0.2],
                },
            ]
        elif difficulty == "medium":
            configs = [
                {
                    "n_samples": 1000,
                    "n_features": 12,
                    "class_sep": 1.2,
                    "weights": [0.85, 0.15],
                },
            ]
        elif difficulty == "hard":
            configs = [
                {
                    "n_samples": 1500,
                    "n_features": 20,
                    "class_sep": 0.8,
                    "weights": [0.9, 0.1],
                },
            ]
        elif difficulty == "extreme":
            configs = [
                {
                    "n_samples": 2000,
                    "n_features": 40,
                    "class_sep": 0.4,
                    "weights": [0.97, 0.03],
                },
            ]
        else:
            configs = [
                {
                    "n_samples": 800,
                    "n_features": 10,
                    "class_sep": 1.0,
                    "weights": [0.8, 0.2],
                }
            ]

        datasets: list[dict[str, Any]] = []
        for idx, config in enumerate(configs):
            X, y = make_classification(
                random_state=42 + idx,
                n_informative=max(2, config["n_features"] // 2),
                n_redundant=config["n_features"] // 4,
                flip_y=0.02 if difficulty in {"hard", "extreme"} else 0.0,
                **config,
            )
            flip_y = 0.02 if difficulty in {"hard", "extreme"} else 0.0
            datasets.append(
                {
                    "name": f"{difficulty}_synthetic_{idx}",
                    "data": X,
                    "target": y,
                    "minority_label": infer_minority_label(y),
                    "difficulty": difficulty,
                    "provenance": synthetic_provenance(
                        "sklearn.datasets.make_classification",
                        random_state=42 + idx,
                        n_informative=max(2, config["n_features"] // 2),
                        n_redundant=config["n_features"] // 4,
                        flip_y=flip_y,
                        **config,
                    ),
                }
            )
        return datasets

load_research_datasets(domains=None, max_samples=10000, include_openml=False)

Load curated datasets for benchmarking.

Parameters:

Name	Type	Description	Default
domains	Sequence[str] | None	Domain names to load.	None
max_samples	int	Maximum number of samples per dataset.	10000
include_openml	bool	Whether to attempt OpenML downloads.	False

Returns:

Type	Description
list[dict[str, Any]]	List of dataset descriptors.

Source code in src/oversampleqa/advanced_benchmark.py

def load_research_datasets(
    self,
    domains: Sequence[str] | None = None,
    max_samples: int = 10_000,
    include_openml: bool = False,
) -> list[dict[str, Any]]:
    """Load curated datasets for benchmarking.

    Args:
        domains: Domain names to load.
        max_samples: Maximum number of samples per dataset.
        include_openml: Whether to attempt OpenML downloads.

    Returns:
        List of dataset descriptors.
    """
    domains = tuple(domains or ("medical", "financial"))
    datasets: list[dict[str, Any]] = []
    for domain in domains:
        datasets.extend(self._load_domain(domain, max_samples, include_openml))
    return datasets

create_synthetic_benchmark_suite(difficulty_levels=None)

Generate synthetic datasets for the requested difficulty levels.

Parameters:

Name	Type	Description	Default
difficulty_levels	Sequence[str] | None	Difficulty labels to generate.	None

Returns:

Type	Description
list[dict[str, Any]]	List of synthetic dataset descriptors.

Source code in src/oversampleqa/advanced_benchmark.py

def create_synthetic_benchmark_suite(
    self, difficulty_levels: Sequence[str] | None = None
) -> list[dict[str, Any]]:
    """Generate synthetic datasets for the requested difficulty levels.

    Args:
        difficulty_levels: Difficulty labels to generate.

    Returns:
        List of synthetic dataset descriptors.
    """
    difficulty_levels = tuple(
        difficulty_levels or ("easy", "medium", "hard", "extreme")
    )
    synthetic: list[dict[str, Any]] = []
    for difficulty in difficulty_levels:
        synthetic.extend(self._generate_difficulty(difficulty))
    return synthetic

StatisticalBenchmark

Advanced benchmarking engine with statistical analysis.

Source code in src/oversampleqa/advanced_benchmark.py

class StatisticalBenchmark:
    """Advanced benchmarking engine with statistical analysis."""

    def __init__(
        self,
        n_folds: int = 5,
        n_repeats: int = 5,
        confidence_level: float = 0.95,
        correction_method: str = "holm",
        random_state: int | None = 42,
    ) -> None:
        if n_folds < 2:
            raise ValueError("n_folds must be at least 2")
        self.n_folds = n_folds
        self.n_repeats = n_repeats
        self.confidence_level = confidence_level
        self.correction_method = correction_method
        self.random_state = random_state
        self._skipped: list[str] = []
        self._fold_records_all: list[FoldRecord] = []

    def run_comprehensive_benchmark(
        self,
        datasets: Sequence[dict[str, Any]],
        oversamplers: Sequence[Any],
        metrics: Sequence[str] | None = None,
    ) -> pd.DataFrame:
        """Run repeated stratified benchmarking across datasets.

        Parameters
        ----------
        datasets:
            Sequence of dataset descriptors. Each entry should provide ``data``,
            ``target`` and optionally ``name`` and ``minority_label``.
        oversamplers:
            Sequence of initialised oversampler instances (will be cloned).
        metrics:
            Distance metrics to evaluate. Defaults to Hassanat, Euclidean, Mahalanobis.
        """

        # mahalanobis is not a default. Without a covariance inverse it is
        # Euclidean, so it produced a third set of rows identical to the
        # euclidean ones -- inflating euclidean's weight in the rankings and
        # splitting one comparison into two for the p-value correction. Pass it
        # explicitly, with cov_inv in metric_kwargs, if you want it.
        metrics = tuple(metrics or ("hassanat", "euclidean"))
        # Reset per run: a reused engine must not accumulate skips or folds.
        self._skipped = []
        self._fold_records_all = []

        all_results: list[BenchmarkResult] = []
        for dataset in datasets:
            all_results.extend(
                self._benchmark_single_dataset(dataset, oversamplers, metrics)
            )

        frame = pd.DataFrame([self._result_to_dict(r) for r in all_results])

        if self._skipped:
            warnings.warn(
                f"{len(self._skipped)} of "
                f"{len(datasets) * len(oversamplers) * len(metrics)} "
                "dataset/oversampler/metric combinations produced no usable "
                "folds and are absent from the results: "
                + ", ".join(self._skipped[:5])
                + (" ..." if len(self._skipped) > 5 else "")
                + ". The most common cause is a minority class too small to "
                "hold out from once it has been split into folds -- try fewer "
                "folds or a larger hidden_ratio.",
                UserWarning,
                stacklevel=2,
            )

        if frame.empty:
            # Return the expected columns rather than a (0, 0) frame. An empty
            # frame with no columns raises KeyError on any column access, so a
            # caller that handles "no results" still breaks.
            return pd.DataFrame(columns=list(_RESULT_COLUMNS))

        frame = self._add_statistical_analysis(frame)
        return frame

    def _benchmark_single_dataset(
        self,
        dataset: dict[str, Any],
        oversamplers: Sequence[Any],
        metrics: Sequence[str],
    ) -> list[BenchmarkResult]:
        """Run benchmark for a single dataset across oversamplers and metrics.

        Args:
            dataset: Dataset descriptor with ``data`` and ``target`` arrays.
            oversamplers: Oversamplers to evaluate.
            metrics: Distance metrics to test.

        Returns:
            List of BenchmarkResult entries for the dataset.
        """
        X, y = dataset["data"], dataset["target"]
        dataset_name = dataset.get("name", "dataset")
        minority_label = dataset.get("minority_label", 1)

        # basic preprocessing to avoid scaling issues for distance metrics
        scaler = StandardScaler()
        X_scaled = scaler.fit_transform(X)

        results: list[BenchmarkResult] = []
        for oversampler in oversamplers:
            for metric in metrics:
                fold_records = self._fold_records(
                    X_scaled,
                    y,
                    minority_label=minority_label,
                    oversampler=oversampler,
                    metric=metric,
                    dataset_name=dataset_name,
                )
                self._fold_records_all.extend(fold_records)
                error_rates = [
                    record.error_rate for record in fold_records if not record.skipped
                ]
                if not error_rates:
                    # Every fold failed for this combination. The per-fold
                    # warnings above explain why, but on a real sweep there are
                    # hundreds of them; record the combination so the caller
                    # gets one summary rather than a silently missing row.
                    self._skipped.append(
                        f"{dataset_name} / {oversampler.__class__.__name__} / {metric}"
                    )
                    continue
                mean_error = float(np.mean(error_rates))
                std_error = (
                    float(np.std(error_rates, ddof=1)) if len(error_rates) > 1 else 0.0
                )
                ci_lower, ci_upper = self._confidence_interval(error_rates)
                results.append(
                    BenchmarkResult(
                        dataset_name=dataset_name,
                        oversampler_name=oversampler.__class__.__name__,
                        metric=metric,
                        hidden_ratio=_FOLD_HIDDEN_RATIO,
                        reference="hidden_minority",
                        minority_label=minority_label,
                        random_state=self.random_state,
                        n_folds=self.n_folds,
                        n_repeats=self.n_repeats,
                        oversampleqa_version=_PACKAGE_VERSION,
                        error_rates=error_rates,
                        mean_error=mean_error,
                        std_error=std_error,
                        confidence_interval=(ci_lower, ci_upper),
                    )
                )
        return results

    def fold_results(self) -> pd.DataFrame:
        """Return one row per attempted fold from the most recent run.

        The summary frame reports a mean and interval per
        (dataset, oversampler, metric). That is enough to read a ranking and
        not enough to check one: it cannot be re-aggregated, plotted as a
        distribution, or given a different interval, and it does not say how
        many folds actually contributed.

        This frame answers those. Skipped folds are present with ``error_rate``
        of ``nan``, ``skipped`` true and a stated ``skip_reason``, because a
        mean over three surviving folds of twenty-five is indistinguishable
        from a mean over twenty-five once the skips are dropped.

        ``split_seed`` is the seed given to the fold splitter for that repeat,
        so a single repeat can be reproduced without rerunning the sweep.

        Returns:
            A long-format frame with :data:`_FOLD_COLUMNS`. Empty of rows but
            not of columns when no run has happened yet, so column access works
            either way.
        """
        frame = pd.DataFrame([asdict(record) for record in self._fold_records_all])
        # Built from the records first, then aliased: FoldRecord stays the one
        # place a fold's identity is defined, rather than carrying each name
        # twice and letting the two drift.
        if frame.empty:
            frame = pd.DataFrame(columns=list(_FOLD_COLUMNS))
        else:
            frame["dataset"] = frame["dataset_name"]
            frame["oversampler"] = frame["oversampler_name"]
        return frame.reindex(columns=list(_FOLD_COLUMNS))

    def _fold_records(
        self,
        X: np.ndarray,
        y: np.ndarray,
        minority_label: int,
        oversampler: Any,
        metric: str,
        dataset_name: str,
    ) -> list[FoldRecord]:
        """Evaluate every repeat/fold, recording skips as well as measurements.

        Args:
            X: Feature matrix.
            y: Target labels.
            minority_label: Minority class label.
            oversampler: Oversampler instance to clone per fold.
            metric: Distance metric name.
            dataset_name: Name recorded on each row.

        Returns:
            One :class:`FoldRecord` per fold attempted, including skipped ones.
        """
        records: list[FoldRecord] = []
        rng = np.random.default_rng(self.random_state)
        name = oversampler.__class__.__name__

        def record(
            repeat: int, fold: int, seed: int | None, error: float, reason: str
        ) -> None:
            records.append(
                FoldRecord(
                    dataset_name=dataset_name,
                    oversampler_name=name,
                    metric=metric,
                    repeat=repeat,
                    fold=fold,
                    split_seed=seed,
                    hidden_ratio=_FOLD_HIDDEN_RATIO,
                    reference="hidden_minority",
                    minority_label=minority_label,
                    random_state=self.random_state,
                    n_folds=self.n_folds,
                    n_repeats=self.n_repeats,
                    oversampleqa_version=_PACKAGE_VERSION,
                    error_rate=error,
                    skipped=bool(reason),
                    skip_reason=reason,
                )
            )

        for repeat in range(self.n_repeats):
            split_seed = (
                None if self.random_state is None else int(rng.integers(0, 1_000_000))
            )
            cv = StratifiedKFold(
                n_splits=self.n_folds,
                shuffle=True,
                random_state=split_seed,
            )
            for fold, (train_idx, _val_idx) in enumerate(cv.split(X, y)):
                # Only the training fold is used. validate_oversampling performs
                # its own hold-out internally, so the CV validation fold has no
                # role here -- the splitter is effectively a stratified
                # subsampler, and each "fold" is one subsample of the data
                # rather than a held-out evaluation. See docs/benchmarking.rst
                # for what the resulting intervals therefore describe.
                X_train = X[train_idx]
                y_train = y[train_idx]
                if len(np.unique(y_train)) < 2:
                    warnings.warn(
                        "Training fold lacks class diversity; skipping fold.",
                        stacklevel=2,
                    )
                    record(
                        repeat,
                        fold,
                        split_seed,
                        float("nan"),
                        "training fold lacks class diversity",
                    )
                    continue
                sampler = clone(oversampler)
                try:
                    error = validate_oversampling(
                        X_train,
                        y_train,
                        minority_label=minority_label,
                        oversampler=sampler,
                        hidden_ratio=_FOLD_HIDDEN_RATIO,
                        metric=metric,
                    )
                except Exception as exc:  # pragma: no cover - defensive
                    warnings.warn(
                        f"Validation failed for fold {fold}: {exc}", stacklevel=2
                    )
                    record(repeat, fold, split_seed, float("nan"), str(exc))
                    continue
                if np.isnan(error):
                    # validate_oversampling returns nan when the sampler made
                    # no synthetic points. This used to `continue` with no
                    # warning at all, so the fold vanished from both the mean
                    # and the count backing the interval.
                    record(
                        repeat,
                        fold,
                        split_seed,
                        float("nan"),
                        "no synthetic samples generated",
                    )
                    continue
                record(repeat, fold, split_seed, float(error), "")
        return records

    def _confidence_interval(self, values: Sequence[float]) -> tuple[float, float]:
        """Return a confidence interval for the **mean** of the provided values.

        Always a confidence interval for the mean, at every sample size.

        This previously switched formula at n = 30: a Student-t interval for the
        mean below, and the 2.5th-97.5th percentiles of the observations at or
        above. Those are different quantities. The t-interval narrows as
        1/sqrt(n); the percentile range describes the spread of individual
        observations and does not narrow at all. Both were written into the same
        ``ci_lower`` / ``ci_upper`` columns, so on sigma = 0.05 data the reported
        width jumped from 0.036 at n = 29 to 0.172 at n = 30 -- 4.7x wider from
        one extra observation -- and intervals could not be compared across
        configurations with different fold counts.

        The t-interval is used throughout. It is exact for normally distributed
        values and asymptotically valid otherwise, and it is what a reader
        assumes a "confidence interval" means.

        Args:
            values: Sample values.

        Returns:
            Lower and upper bounds for the mean, at the configured level.
        """
        if len(values) < 2:
            return (
                float(values[0]) if values else 0.0,
                float(values[0]) if values else 0.0,
            )
        arr = np.asarray(values, dtype=float)
        mean = float(arr.mean())
        alpha = 1 - self.confidence_level
        standard_error = float(stats.sem(arr))
        if standard_error == 0.0:
            # Every observation identical: the mean is known exactly.
            return (mean, mean)
        margin = float(stats.t.ppf(1 - alpha / 2, len(arr) - 1)) * standard_error
        return (mean - margin, mean + margin)

    def _add_statistical_analysis(self, frame: pd.DataFrame) -> pd.DataFrame:
        """Add pairwise p-values and effect sizes per dataset.

        Args:
            frame: Benchmark results dataframe.

        Returns:
            Dataframe with pairwise statistics columns populated.
        """
        frame = frame.copy()
        frame["pairwise_p_values"] = None
        frame["pairwise_effect_sizes"] = None

        # Grouped by (dataset, metric), not dataset alone. Grouping by dataset
        # put several metrics in one slice, and the lookups below take .iloc[0]
        # -- so the tests ran on whichever metric happened to sort first and the
        # result was stamped onto every row, including rows for the other
        # metrics. Error rates are not comparable across metrics, so those
        # p-values described a comparison the row did not represent.
        for _key, group in frame.groupby(
            ["dataset_name", "metric"], sort=False
        ):
            if len(group) < 2:
                continue
            pvals = self._pairwise_statistical_tests(group)
            effects = self._calculate_effect_sizes(group)
            for idx in group.index:
                # strict_json_dumps, not json.dumps: a skipped fold leaves a
                # nan in error_rates, Wilcoxon then returns nan, and the plain
                # encoder writes a bare `NaN` token. That is not JSON -- a
                # strict parser rejects the column outright -- and these two
                # columns are the only export path that had bypassed the
                # helper. nan becomes null, which parsers accept and which says
                # the same thing: no comparison was available.
                frame.at[idx, "pairwise_p_values"] = strict_json_dumps(
                    pvals, indent=None
                )
                frame.at[idx, "pairwise_effect_sizes"] = strict_json_dumps(
                    effects, indent=None
                )
        return frame

    def _pairwise_statistical_tests(
        self, dataset_slice: pd.DataFrame
    ) -> dict[str, float]:
        """Compute pairwise Wilcoxon tests across oversamplers.

        Args:
            dataset_slice: Subset of results for a single dataset.

        Returns:
            Mapping of ``oversampler_a_vs_b`` to corrected p-values.
        """
        p_values: dict[str, float] = {}
        oversamplers = dataset_slice["oversampler_name"].unique()
        for i, os1 in enumerate(oversamplers):
            for os2 in oversamplers[i + 1 :]:
                errors1 = np.asarray(
                    dataset_slice.loc[
                        dataset_slice["oversampler_name"] == os1, "error_rates"
                    ].iloc[0],
                    dtype=float,
                )
                errors2 = np.asarray(
                    dataset_slice.loc[
                        dataset_slice["oversampler_name"] == os2, "error_rates"
                    ].iloc[0],
                    dtype=float,
                )
                try:
                    if (
                        len(errors1) == 0
                        or len(errors2) == 0
                        or len(errors1) != len(errors2)
                        or np.allclose(errors1, errors2)
                    ):
                        p_val = 1.0
                    else:
                        with warnings.catch_warnings():
                            warnings.simplefilter("ignore", RuntimeWarning)
                            _, p_val = stats.wilcoxon(errors1, errors2)
                except Exception:
                    p_val = 1.0
                key = f"{os1}_vs_{os2}"
                p_values[key] = p_val
        return self._apply_correction(p_values)

    def _calculate_effect_sizes(self, dataset_slice: pd.DataFrame) -> dict[str, float]:
        """Compute pairwise matched-pairs rank-biserial correlations.

        Paired, to match the design the p-value comes from. The tests are
        Wilcoxon signed-rank, which pairs the two samplers fold by fold; the
        effect size was independent-samples Cohen's d, which pools the two
        standard deviations and throws that pairing away. On fold errors that
        move together -- an awkward fold is awkward for both samplers -- the
        pooled deviation is dominated by between-fold variation that the paired
        test has already removed, so the effect looks smaller than the test
        says it is.

        Rank-biserial is the natural companion to a rank test: it is computed
        from the same signed ranks Wilcoxon uses.

        Args:
            dataset_slice: Results for one (dataset, metric).

        Returns:
            Mapping of ``a_vs_b`` to a correlation in ``[-1, 1]``. Positive
            means the first sampler had the higher error rate on more folds,
            weighted by how much higher -- so positive favours the second.
        """
        effect_sizes: dict[str, float] = {}
        oversamplers = dataset_slice["oversampler_name"].unique()
        for i, os1 in enumerate(oversamplers):
            for os2 in oversamplers[i + 1 :]:
                errors1 = np.asarray(
                    dataset_slice.loc[
                        dataset_slice["oversampler_name"] == os1, "error_rates"
                    ].iloc[0],
                    dtype=float,
                )
                errors2 = np.asarray(
                    dataset_slice.loc[
                        dataset_slice["oversampler_name"] == os2, "error_rates"
                    ].iloc[0],
                    dtype=float,
                )
                effect = self._rank_biserial(errors1, errors2)
                if effect is not None:
                    effect_sizes[f"{os1}_vs_{os2}"] = effect
        return effect_sizes

    @staticmethod
    def _rank_biserial(x: np.ndarray, y: np.ndarray) -> float | None:
        """Matched-pairs rank-biserial correlation for two paired samples.

        ``(W+ - W-) / (W+ + W-)`` over the signed ranks of the differences,
        which is exactly what Wilcoxon signed-rank sums. Zero differences are
        dropped, as the test drops them.

        Returns ``None`` when the samples cannot be paired or every difference
        is zero -- there is no effect to report, and 0.0 would claim there is
        one and that it is exactly nil.
        """
        if len(x) != len(y) or len(x) == 0:
            return None
        diff = x - y
        nonzero = diff[diff != 0]
        if nonzero.size == 0:
            return None
        ranks = stats.rankdata(np.abs(nonzero))
        positive = float(ranks[nonzero > 0].sum())
        negative = float(ranks[nonzero < 0].sum())
        total = positive + negative
        if total == 0:
            return None
        return (positive - negative) / total

    @staticmethod
    def _pooled_std(x: np.ndarray, y: np.ndarray) -> float:
        """Return pooled standard deviation for two samples.

        Args:
            x: Sample 1.
            y: Sample 2.

        Returns:
            Pooled standard deviation.
        """
        if len(x) < 2 or len(y) < 2:
            return 0.0
        n1, n2 = len(x), len(y)
        s1, s2 = x.var(ddof=1), y.var(ddof=1)
        pooled = ((n1 - 1) * s1 + (n2 - 1) * s2) / (n1 + n2 - 2)
        return math.sqrt(max(pooled, 0.0))

    def _apply_correction(self, p_values: dict[str, float]) -> dict[str, float]:
        """Apply multiple-comparison correction to p-values.

        With 8 oversamplers there are 28 pairwise comparisons, and uncorrected
        p-values manufacture significance at that many looks.

        Args:
            p_values: Raw p-values keyed by comparison.

        Returns:
            Corrected p-values using the configured method: ``"holm"``
            (family-wise error rate, the default and the right choice for small
            families), ``"bh"`` / ``"fdr"`` (false discovery rate, better when
            the family is large and some false positives are tolerable), or
            ``"bonferroni"``.

        Notes:
            Both step procedures enforce **monotonicity**, which the previous
            Holm implementation omitted: it scaled each p-value by its rank
            without taking a running maximum, so raw p-values of
            ``[0.01, 0.02, 0.03]`` corrected to ``[0.03, 0.04, 0.03]``. The
            least significant comparison came out *more* significant than the
            middle one, which is incoherent and can flip a decision at a fixed
            alpha.
        """
        if not p_values:
            return p_values

        keys = list(p_values)
        raw = np.asarray([p_values[k] for k in keys], dtype=float)
        m = len(raw)
        method = self.correction_method.lower()

        if method == "bonferroni":
            adjusted = np.minimum(raw * m, 1.0)
            return dict(zip(keys, (float(v) for v in adjusted), strict=True))

        order = np.argsort(raw, kind="stable")

        if method in {"bh", "fdr", "benjamini-hochberg"}:
            # Step-up: scale by m / rank, then enforce monotonicity from the
            # largest p-value downward.
            ranks = np.arange(1, m + 1)
            scaled = raw[order] * m / ranks
            adjusted_sorted = np.minimum.accumulate(scaled[::-1])[::-1]
        else:
            # Holm step-down: scale by the number of remaining hypotheses, then
            # enforce monotonicity from the smallest p-value upward.
            scaled = raw[order] * (m - np.arange(m))
            adjusted_sorted = np.maximum.accumulate(scaled)

        adjusted_sorted = np.minimum(adjusted_sorted, 1.0)
        adjusted = np.empty_like(adjusted_sorted)
        adjusted[order] = adjusted_sorted
        return dict(zip(keys, (float(v) for v in adjusted), strict=True))

    @staticmethod
    def _result_to_dict(result: BenchmarkResult) -> dict[str, Any]:
        """Convert BenchmarkResult to a serializable dictionary.

        Args:
            result: Benchmark result structure.

        Returns:
            Dict suitable for DataFrame construction or serialization.
        """
        return {
            # Canonical names first; the `_name` spellings below are this
            # frame's originals and are kept for existing readers.
            "dataset": result.dataset_name,
            "oversampler": result.oversampler_name,
            "dataset_name": result.dataset_name,
            "oversampler_name": result.oversampler_name,
            "metric": result.metric,
            "hidden_ratio": result.hidden_ratio,
            "reference": result.reference,
            "minority_label": result.minority_label,
            "random_state": result.random_state,
            "n_folds": result.n_folds,
            "n_repeats": result.n_repeats,
            "oversampleqa_version": result.oversampleqa_version,
            "mean_error": result.mean_error,
            "std_error": result.std_error,
            "ci_lower": result.confidence_interval[0],
            "ci_upper": result.confidence_interval[1],
            "n_observations": len(result.error_rates),
            "error_rates": result.error_rates,
            "effect_size": result.effect_size,
            "p_value": result.p_value,
        }

run_comprehensive_benchmark(datasets, oversamplers, metrics=None)

Run repeated stratified benchmarking across datasets.

Parameters

datasets: Sequence of dataset descriptors. Each entry should provide data, target and optionally name and minority_label. oversamplers: Sequence of initialised oversampler instances (will be cloned). metrics: Distance metrics to evaluate. Defaults to Hassanat, Euclidean, Mahalanobis.

Source code in src/oversampleqa/advanced_benchmark.py

def run_comprehensive_benchmark(
    self,
    datasets: Sequence[dict[str, Any]],
    oversamplers: Sequence[Any],
    metrics: Sequence[str] | None = None,
) -> pd.DataFrame:
    """Run repeated stratified benchmarking across datasets.

    Parameters
    ----------
    datasets:
        Sequence of dataset descriptors. Each entry should provide ``data``,
        ``target`` and optionally ``name`` and ``minority_label``.
    oversamplers:
        Sequence of initialised oversampler instances (will be cloned).
    metrics:
        Distance metrics to evaluate. Defaults to Hassanat, Euclidean, Mahalanobis.
    """

    # mahalanobis is not a default. Without a covariance inverse it is
    # Euclidean, so it produced a third set of rows identical to the
    # euclidean ones -- inflating euclidean's weight in the rankings and
    # splitting one comparison into two for the p-value correction. Pass it
    # explicitly, with cov_inv in metric_kwargs, if you want it.
    metrics = tuple(metrics or ("hassanat", "euclidean"))
    # Reset per run: a reused engine must not accumulate skips or folds.
    self._skipped = []
    self._fold_records_all = []

    all_results: list[BenchmarkResult] = []
    for dataset in datasets:
        all_results.extend(
            self._benchmark_single_dataset(dataset, oversamplers, metrics)
        )

    frame = pd.DataFrame([self._result_to_dict(r) for r in all_results])

    if self._skipped:
        warnings.warn(
            f"{len(self._skipped)} of "
            f"{len(datasets) * len(oversamplers) * len(metrics)} "
            "dataset/oversampler/metric combinations produced no usable "
            "folds and are absent from the results: "
            + ", ".join(self._skipped[:5])
            + (" ..." if len(self._skipped) > 5 else "")
            + ". The most common cause is a minority class too small to "
            "hold out from once it has been split into folds -- try fewer "
            "folds or a larger hidden_ratio.",
            UserWarning,
            stacklevel=2,
        )

    if frame.empty:
        # Return the expected columns rather than a (0, 0) frame. An empty
        # frame with no columns raises KeyError on any column access, so a
        # caller that handles "no results" still breaks.
        return pd.DataFrame(columns=list(_RESULT_COLUMNS))

    frame = self._add_statistical_analysis(frame)
    return frame

fold_results()

Return one row per attempted fold from the most recent run.

The summary frame reports a mean and interval per (dataset, oversampler, metric). That is enough to read a ranking and not enough to check one: it cannot be re-aggregated, plotted as a distribution, or given a different interval, and it does not say how many folds actually contributed.

This frame answers those. Skipped folds are present with error_rate of nan, skipped true and a stated skip_reason, because a mean over three surviving folds of twenty-five is indistinguishable from a mean over twenty-five once the skips are dropped.

split_seed is the seed given to the fold splitter for that repeat, so a single repeat can be reproduced without rerunning the sweep.

Returns:

Type	Description
DataFrame	A long-format frame with :data:_FOLD_COLUMNS. Empty of rows but
DataFrame	not of columns when no run has happened yet, so column access works
DataFrame	either way.

Source code in src/oversampleqa/advanced_benchmark.py

def fold_results(self) -> pd.DataFrame:
    """Return one row per attempted fold from the most recent run.

    The summary frame reports a mean and interval per
    (dataset, oversampler, metric). That is enough to read a ranking and
    not enough to check one: it cannot be re-aggregated, plotted as a
    distribution, or given a different interval, and it does not say how
    many folds actually contributed.

    This frame answers those. Skipped folds are present with ``error_rate``
    of ``nan``, ``skipped`` true and a stated ``skip_reason``, because a
    mean over three surviving folds of twenty-five is indistinguishable
    from a mean over twenty-five once the skips are dropped.

    ``split_seed`` is the seed given to the fold splitter for that repeat,
    so a single repeat can be reproduced without rerunning the sweep.

    Returns:
        A long-format frame with :data:`_FOLD_COLUMNS`. Empty of rows but
        not of columns when no run has happened yet, so column access works
        either way.
    """
    frame = pd.DataFrame([asdict(record) for record in self._fold_records_all])
    # Built from the records first, then aliased: FoldRecord stays the one
    # place a fold's identity is defined, rather than carrying each name
    # twice and letting the two drift.
    if frame.empty:
        frame = pd.DataFrame(columns=list(_FOLD_COLUMNS))
    else:
        frame["dataset"] = frame["dataset_name"]
        frame["oversampler"] = frame["oversampler_name"]
    return frame.reindex(columns=list(_FOLD_COLUMNS))

ValidationCache

Caching layer for validation results and distance computations.

Caching is opt-in. Constructing this class is the caller's decision; nothing in the package builds one at import time, and no directory is created until the first write.

.. warning::

Not thread-safe across instances, and not process-safe. A single instance guards its own in-memory bookkeeping with a lock, so concurrent reads and writes through one instance will not corrupt its accounting. joblib on-disk writes are not atomic, so two processes (or two instances pointed at the same directory) writing the same key can interleave and leave a truncated file. Give each process its own cache_dir.

.. note::

Whether caching pays depends entirely on how expensive the metric is relative to hashing its inputs. Content hashing must read every input byte, so for a BLAS-backed metric such as euclidean the cache is a net loss; for hassanat it is worth tens of times the compute. See :doc:/reproducibility.

Parameters

cache_dir : str or Path, optional Where to store cached artefacts. Defaults to the per-user cache directory, never the working directory. max_entries : int, default=128 Upper bound on in-memory distance matrices. Least-recently-used entries are evicted first. memory_mb : int, default=1000 Upper bound on the in-memory tier, in megabytes. Enforced: entries are evicted oldest-first until the total fits.

Source code in src/oversampleqa/caching.py

class ValidationCache:
    """Caching layer for validation results and distance computations.

    Caching is **opt-in**. Constructing this class is the caller's decision;
    nothing in the package builds one at import time, and no directory is
    created until the first write.

    .. warning::

       **Not thread-safe across instances, and not process-safe.** A single
       instance guards its own in-memory bookkeeping with a lock, so concurrent
       reads and writes through one instance will not corrupt its accounting.
       ``joblib`` on-disk writes are *not* atomic, so two processes (or two
       instances pointed at the same directory) writing the same key can
       interleave and leave a truncated file. Give each process its own
       ``cache_dir``.

    .. note::

       Whether caching pays depends entirely on how expensive the metric is
       relative to hashing its inputs. Content hashing must read every input
       byte, so for a BLAS-backed metric such as ``euclidean`` the cache is a
       net loss; for ``hassanat`` it is worth tens of times the compute. See
       :doc:`/reproducibility`.

    Parameters
    ----------
    cache_dir : str or Path, optional
        Where to store cached artefacts. Defaults to the per-user cache
        directory, never the working directory.
    max_entries : int, default=128
        Upper bound on in-memory distance matrices. Least-recently-used entries
        are evicted first.
    memory_mb : int, default=1000
        Upper bound on the in-memory tier, in megabytes. Enforced: entries are
        evicted oldest-first until the total fits.
    """

    def __init__(
        self,
        cache_dir: str | Path | None = None,
        memory_mb: int = 1000,
        max_entries: int = 128,
    ) -> None:
        self.cache_dir = (
            Path(cache_dir) if cache_dir is not None else default_cache_dir()
        )
        self.bytes_limit = memory_mb * 1024 * 1024
        self.max_entries = max_entries
        self._memory: joblib.Memory | None = None
        self._lock = threading.Lock()
        self._store: OrderedDict[str, NDArray[np.floating]] = OrderedDict()
        self._nbytes = 0

    def _ensure_dir(self) -> None:
        """Create the cache directory. Called on first write, never on import."""
        self.cache_dir.mkdir(parents=True, exist_ok=True)

    @property
    def memory(self) -> joblib.Memory:
        """Lazily-created joblib store; creates the directory on first use."""
        if self._memory is None:
            self._ensure_dir()
            self._memory = joblib.Memory(self.cache_dir, verbose=0)
        return self._memory

    @property
    def size_bytes(self) -> int:
        """Bytes currently held by the in-memory tier."""
        with self._lock:
            return self._nbytes

    def clear(self) -> None:
        """Drop everything held in memory. Does not touch the disk store."""
        with self._lock:
            self._store.clear()
            self._nbytes = 0

    def get_data_hash(self, X: NDArray[Any], y: NDArray[Any]) -> str:
        """Return stable SHA256 hash for dataset.

        Args:
            X: Feature matrix.
            y: Target labels.

        Returns:
            SHA256 hex digest.
        """
        hasher = hashlib.sha256()
        self._update_hasher(hasher, X)
        self._update_hasher(hasher, y)
        return hasher.hexdigest()

    def cache_validation_result(self, params_hash: str, result: float) -> None:
        """Persist validation result using joblib.

        Args:
            params_hash: Cache key for the run parameters.
            result: Error rate to persist.
        """
        self._ensure_dir()
        path = self.cache_dir / f"validation_{params_hash}.pkl"
        joblib.dump(result, path)

    def load_validation_result(self, params_hash: str) -> float | None:
        """Retrieve cached validation result if present.

        Args:
            params_hash: Cache key for the run parameters.

        Returns:
            Cached error rate if available.
        """
        path = self.cache_dir / f"validation_{params_hash}.pkl"
        if path.exists():
            cached: float = joblib.load(path)
            return cached
        return None

    def cached_distance_matrix(
        self,
        optimizer: OptimizedDistanceMatrix,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        metric: str,
        batch_size: int | str = "auto",
        **kwargs: Any,
    ) -> NDArray[np.floating]:
        """Return cached distance matrix or compute and cache it.

        The returned array is **read-only**. Cache hits hand back the stored
        array rather than a copy, so an in-place operation downstream would
        otherwise corrupt every later hit silently; the write flag turns that
        into a loud ``ValueError`` instead. Call ``.copy()`` if you need to
        modify it.

        ``batch_size`` is deliberately **not** part of the key: batching splits
        the same computation into chunks and concatenates them, so it cannot
        change the result. ``test_caching.py`` pins that invariant for every
        registered metric.

        Args:
            optimizer: OptimizedDistanceMatrix instance. Used only to compute a
                miss -- it is never part of the cache key.
            X1: First feature matrix.
            X2: Second feature matrix.
            metric: Distance metric name.
            batch_size: Batch size or mode.
            **kwargs: Metric keyword arguments.

        Returns:
            Read-only distance matrix.
        """
        key = self._distance_key(X1, X2, metric, kwargs)

        with self._lock:
            hit = self._store.get(key)
            if hit is not None:
                self._store.move_to_end(key)
                return hit

        result = optimizer._compute_uncached(
            X1, X2, metric=metric, batch_size=batch_size, **kwargs
        )
        result.setflags(write=False)
        self._remember(key, result)
        return result

    def _remember(self, key: str, arr: NDArray[np.floating]) -> None:
        """Store ``arr`` under ``key``, evicting until the limits are met."""
        with self._lock:
            if key in self._store:
                self._store.move_to_end(key)
                return
            self._store[key] = arr
            self._nbytes += arr.nbytes
            while self._store and (
                self._nbytes > self.bytes_limit or len(self._store) > self.max_entries
            ):
                _, evicted = self._store.popitem(last=False)
                self._nbytes -= evicted.nbytes
                logger.debug(
                    "Evicted a %d-byte distance matrix; %d bytes still cached",
                    evicted.nbytes,
                    self._nbytes,
                )

    def _distance_key(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        metric: str,
        kwargs: dict[str, Any],
    ) -> str:
        """Return a stable key for distance matrix caching.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.
            metric: Distance metric name.
            kwargs: Metric keyword arguments.

        Returns:
            Cache key as a hex digest.
        """
        hasher = hashlib.sha256()
        self._update_hasher(hasher, X1)
        self._update_hasher(hasher, X2)
        hasher.update(metric.encode("utf-8"))
        if kwargs:
            serialized = pickle.dumps(sorted(kwargs.items(), key=lambda item: item[0]))
            hasher.update(serialized)
        return hasher.hexdigest()

    @staticmethod
    def _update_hasher(hasher: hashlib._Hash, arr: NDArray[Any]) -> None:
        """Update the hasher with array shape, dtype, and data bytes.

        Args:
            hasher: Hash object to update.
            arr: Array to serialize into the hash.
        """
        hasher.update(str(arr.shape).encode("utf-8"))
        hasher.update(str(arr.dtype).encode("utf-8"))
        hasher.update(arr.tobytes(order="C"))

memory property

Lazily-created joblib store; creates the directory on first use.

size_bytes property

Bytes currently held by the in-memory tier.

clear()

Drop everything held in memory. Does not touch the disk store.

Source code in src/oversampleqa/caching.py

def clear(self) -> None:
    """Drop everything held in memory. Does not touch the disk store."""
    with self._lock:
        self._store.clear()
        self._nbytes = 0

get_data_hash(X, y)

Return stable SHA256 hash for dataset.

Parameters:

Name	Type	Description	Default
X	NDArray[Any]	Feature matrix.	required
y	NDArray[Any]	Target labels.	required

Returns:

Type	Description
str	SHA256 hex digest.

Source code in src/oversampleqa/caching.py

def get_data_hash(self, X: NDArray[Any], y: NDArray[Any]) -> str:
    """Return stable SHA256 hash for dataset.

    Args:
        X: Feature matrix.
        y: Target labels.

    Returns:
        SHA256 hex digest.
    """
    hasher = hashlib.sha256()
    self._update_hasher(hasher, X)
    self._update_hasher(hasher, y)
    return hasher.hexdigest()

cache_validation_result(params_hash, result)

Persist validation result using joblib.

Parameters:

Name	Type	Description	Default
params_hash	str	Cache key for the run parameters.	required
result	float	Error rate to persist.	required

Source code in src/oversampleqa/caching.py

def cache_validation_result(self, params_hash: str, result: float) -> None:
    """Persist validation result using joblib.

    Args:
        params_hash: Cache key for the run parameters.
        result: Error rate to persist.
    """
    self._ensure_dir()
    path = self.cache_dir / f"validation_{params_hash}.pkl"
    joblib.dump(result, path)

load_validation_result(params_hash)

Retrieve cached validation result if present.

Parameters:

Name	Type	Description	Default
params_hash	str	Cache key for the run parameters.	required

Returns:

Type	Description
float | None	Cached error rate if available.

Source code in src/oversampleqa/caching.py

def load_validation_result(self, params_hash: str) -> float | None:
    """Retrieve cached validation result if present.

    Args:
        params_hash: Cache key for the run parameters.

    Returns:
        Cached error rate if available.
    """
    path = self.cache_dir / f"validation_{params_hash}.pkl"
    if path.exists():
        cached: float = joblib.load(path)
        return cached
    return None

cached_distance_matrix(optimizer, X1, X2, metric, batch_size='auto', **kwargs)

Return cached distance matrix or compute and cache it.

The returned array is read-only. Cache hits hand back the stored array rather than a copy, so an in-place operation downstream would otherwise corrupt every later hit silently; the write flag turns that into a loud ValueError instead. Call .copy() if you need to modify it.

batch_size is deliberately not part of the key: batching splits the same computation into chunks and concatenates them, so it cannot change the result. test_caching.py pins that invariant for every registered metric.

Parameters:

Name	Type	Description	Default
optimizer	OptimizedDistanceMatrix	OptimizedDistanceMatrix instance. Used only to compute a miss -- it is never part of the cache key.	required
X1	NDArray[floating]	First feature matrix.	required
X2	NDArray[floating]	Second feature matrix.	required
metric	str	Distance metric name.	required
batch_size	int | str	Batch size or mode.	'auto'
**kwargs	Any	Metric keyword arguments.	{}

Returns:

Type	Description
NDArray[floating]	Read-only distance matrix.

Source code in src/oversampleqa/caching.py

def cached_distance_matrix(
    self,
    optimizer: OptimizedDistanceMatrix,
    X1: NDArray[np.floating],
    X2: NDArray[np.floating],
    metric: str,
    batch_size: int | str = "auto",
    **kwargs: Any,
) -> NDArray[np.floating]:
    """Return cached distance matrix or compute and cache it.

    The returned array is **read-only**. Cache hits hand back the stored
    array rather than a copy, so an in-place operation downstream would
    otherwise corrupt every later hit silently; the write flag turns that
    into a loud ``ValueError`` instead. Call ``.copy()`` if you need to
    modify it.

    ``batch_size`` is deliberately **not** part of the key: batching splits
    the same computation into chunks and concatenates them, so it cannot
    change the result. ``test_caching.py`` pins that invariant for every
    registered metric.

    Args:
        optimizer: OptimizedDistanceMatrix instance. Used only to compute a
            miss -- it is never part of the cache key.
        X1: First feature matrix.
        X2: Second feature matrix.
        metric: Distance metric name.
        batch_size: Batch size or mode.
        **kwargs: Metric keyword arguments.

    Returns:
        Read-only distance matrix.
    """
    key = self._distance_key(X1, X2, metric, kwargs)

    with self._lock:
        hit = self._store.get(key)
        if hit is not None:
            self._store.move_to_end(key)
            return hit

    result = optimizer._compute_uncached(
        X1, X2, metric=metric, batch_size=batch_size, **kwargs
    )
    result.setflags(write=False)
    self._remember(key, result)
    return result

OversamplingValidator

Bases: BaseEstimator

Validate an oversampler, following the scikit-learn estimator contract.

Lower scores are better: the score is the hidden-majority error rate, so score returns its negation, matching scikit-learn's "greater is better" convention for scorers.

Parameters

oversampler : object An imbalanced-learn sampler. minority_label : int, optional Minority class. Inferred as the least frequent label when omitted. hidden_ratio : float, default=0.1 Fraction held out. reference : {"hidden_minority", "train_minority"}, default="hidden_minority" Which minority set to compare against. metric : str, default="hassanat" Distance metric. metric_params : dict, optional Extra keyword arguments for the metric. n_repeats : int, default=1 Independent hold-out splits. random_state : int, Generator, SeedSequence or None, default=42 Seeds the split.

Attributes

report_ : ValidationReport Set by :meth:fit. error_rate_ : float Set by :meth:fit.

Notes

The constructor stores its arguments unchanged and does no validation or computation, as scikit-learn requires -- get_params / set_params round-trip, and clone works. All checking happens in :meth:fit.

.. warning::

Cross-validation folds must be large enough to support the estimand. Scoring runs a full validation on each test fold, which holds out hidden_ratio of that fold's minority. With cv=3 on 136 minority points, a test fold has ~45 and a 10% hold-out leaves 4 — below min_hidden, so validation raises.

scikit-learn catches scorer exceptions and records nan, so this surfaces as an all-nan cv_results_ with no explanation. Pass error_score="raise" to see the real message. Either use fewer folds, supply more minority data, or lower min_hidden deliberately.

Examples

Tuning a sampler against synthetic-sample quality becomes two lines::

search = GridSearchCV(
    OversamplingValidator(SMOTE(random_state=0)),
    {"oversampler": [SMOTE(k_neighbors=k) for k in (3, 5, 9)]},
    scoring=validation_scorer,
)
search.fit(X, y)

Source code in src/oversampleqa/estimator.py

class OversamplingValidator(BaseEstimator):
    """Validate an oversampler, following the scikit-learn estimator contract.

    Lower scores are better: the score *is* the hidden-majority error rate, so
    ``score`` returns its negation, matching scikit-learn's "greater is better"
    convention for scorers.

    Parameters
    ----------
    oversampler : object
        An ``imbalanced-learn`` sampler.
    minority_label : int, optional
        Minority class. Inferred as the least frequent label when omitted.
    hidden_ratio : float, default=0.1
        Fraction held out.
    reference : {"hidden_minority", "train_minority"}, default="hidden_minority"
        Which minority set to compare against.
    metric : str, default="hassanat"
        Distance metric.
    metric_params : dict, optional
        Extra keyword arguments for the metric.
    n_repeats : int, default=1
        Independent hold-out splits.
    random_state : int, Generator, SeedSequence or None, default=42
        Seeds the split.

    Attributes
    ----------
    report_ : ValidationReport
        Set by :meth:`fit`.
    error_rate_ : float
        Set by :meth:`fit`.

    Notes
    -----
    The constructor stores its arguments unchanged and does no validation or
    computation, as scikit-learn requires -- ``get_params`` / ``set_params``
    round-trip, and ``clone`` works. All checking happens in :meth:`fit`.

    .. warning::

       **Cross-validation folds must be large enough to support the estimand.**
       Scoring runs a full validation on each test fold, which holds out
       ``hidden_ratio`` of *that fold's* minority. With ``cv=3`` on 136 minority
       points, a test fold has ~45 and a 10% hold-out leaves 4 — below
       ``min_hidden``, so validation raises.

       scikit-learn catches scorer exceptions and records ``nan``, so this
       surfaces as an all-``nan`` ``cv_results_`` with no explanation. Pass
       ``error_score="raise"`` to see the real message. Either use fewer folds,
       supply more minority data, or lower ``min_hidden`` deliberately.

    Examples
    --------
    Tuning a sampler against synthetic-sample quality becomes two lines::

        search = GridSearchCV(
            OversamplingValidator(SMOTE(random_state=0)),
            {"oversampler": [SMOTE(k_neighbors=k) for k in (3, 5, 9)]},
            scoring=validation_scorer,
        )
        search.fit(X, y)
    """

    def __init__(
        self,
        oversampler: Any,
        *,
        minority_label: int | None = None,
        hidden_ratio: float = 0.1,
        reference: ReferenceSet = "hidden_minority",
        metric: str = "hassanat",
        metric_params: dict[str, Any] | None = None,
        n_repeats: int = 1,
        random_state: RandomStateLike = 42,
    ) -> None:
        # Stored unchanged. No validation here: scikit-learn requires that
        # __init__ be a pure assignment so clone() and set_params() behave.
        self.oversampler = oversampler
        self.minority_label = minority_label
        self.hidden_ratio = hidden_ratio
        self.reference = reference
        self.metric = metric
        self.metric_params = metric_params
        self.n_repeats = n_repeats
        self.random_state = random_state

    def _resolve_minority_label(self, y: NDArray[np.integer]) -> int:
        """Infer the minority label as the least frequent one."""
        if self.minority_label is not None:
            return int(self.minority_label)
        labels, counts = np.unique(y, return_counts=True)
        return int(labels[int(np.argmin(counts))])

    def fit(
        self, X: NDArray[np.floating], y: NDArray[np.integer]
    ) -> OversamplingValidator:
        """Run validation and store the report.

        Args:
            X: Feature matrix.
            y: Target labels.

        Returns:
            self, so calls chain.

        Raises:
            ValidationError: If the inputs cannot support validation.
        """
        from .validator import validate_oversampling

        X = np.asarray(X, dtype=float)
        y = np.asarray(y)
        if X.ndim != 2:
            raise ValidationError(f"X must be 2-D; got shape {X.shape}")
        if len(X) != len(y):
            raise ValidationError(f"X has {len(X)} rows but y has {len(y)}")

        minority_label = self._resolve_minority_label(y)

        details = validate_oversampling(
            X,
            y,
            minority_label,
            self.oversampler,
            hidden_ratio=self.hidden_ratio,
            metric=self.metric,
            metric_kwargs=self.metric_params,
            return_details=True,
            reference=self.reference,
            n_repeats=self.n_repeats,
            random_state=self.random_state,
        )

        # return_details=True always yields ValidationDetails.
        if not isinstance(details, ValidationDetails):  # pragma: no cover
            raise ValidationError(
                "validate_oversampling(return_details=True) must return "
                "ValidationDetails"
            )
        self.error_rate_ = float(details.error_rate)
        self.report_ = ValidationReport(
            error_rate=self.error_rate_,
            metadata=RunMetadata.capture(
                X,
                y,
                self.oversampler,
                minority_label=minority_label,
                metric=self.metric,
                hidden_ratio=self.hidden_ratio,
                reference=self.reference,
                random_state=(
                    self.random_state
                    if isinstance(self.random_state, (int, type(None)))
                    else None
                ),
                n_repeats=self.n_repeats,
            ),
            details=details,
        )
        self.minority_label_ = minority_label
        return self

    def score(
        self,
        X: NDArray[np.floating] | None = None,
        y: NDArray[np.integer] | None = None,
    ) -> float:
        """Return the negated error rate, so greater is better.

        Scikit-learn's convention is that a higher score is better, but a
        higher error rate is worse. Returning the raw rate would make
        ``GridSearchCV`` select the *worst* sampler, so it is negated here.

        Args:
            X: Optional data to validate instead of the fitted run.
            y: Labels matching ``X``.

        Returns:
            Negated error rate.
        """
        if X is not None and y is not None:
            # deep=False: nested `oversampler__*` keys are not constructor
            # arguments, which is why sklearn's own clone() uses shallow params.
            fresh = self.__class__(**self.get_params(deep=False))
            return -float(fresh.fit(X, y).error_rate_)
        if not hasattr(self, "error_rate_"):
            raise ValidationError("call fit before score, or pass X and y")
        return -self.error_rate_

fit(X, y)

Run validation and store the report.

Parameters:

Name	Type	Description	Default
X	NDArray[floating]	Feature matrix.	required
y	NDArray[integer]	Target labels.	required

Returns:

Type	Description
OversamplingValidator	self, so calls chain.

Raises:

Type	Description
ValidationError	If the inputs cannot support validation.

Source code in src/oversampleqa/estimator.py

def fit(
    self, X: NDArray[np.floating], y: NDArray[np.integer]
) -> OversamplingValidator:
    """Run validation and store the report.

    Args:
        X: Feature matrix.
        y: Target labels.

    Returns:
        self, so calls chain.

    Raises:
        ValidationError: If the inputs cannot support validation.
    """
    from .validator import validate_oversampling

    X = np.asarray(X, dtype=float)
    y = np.asarray(y)
    if X.ndim != 2:
        raise ValidationError(f"X must be 2-D; got shape {X.shape}")
    if len(X) != len(y):
        raise ValidationError(f"X has {len(X)} rows but y has {len(y)}")

    minority_label = self._resolve_minority_label(y)

    details = validate_oversampling(
        X,
        y,
        minority_label,
        self.oversampler,
        hidden_ratio=self.hidden_ratio,
        metric=self.metric,
        metric_kwargs=self.metric_params,
        return_details=True,
        reference=self.reference,
        n_repeats=self.n_repeats,
        random_state=self.random_state,
    )

    # return_details=True always yields ValidationDetails.
    if not isinstance(details, ValidationDetails):  # pragma: no cover
        raise ValidationError(
            "validate_oversampling(return_details=True) must return "
            "ValidationDetails"
        )
    self.error_rate_ = float(details.error_rate)
    self.report_ = ValidationReport(
        error_rate=self.error_rate_,
        metadata=RunMetadata.capture(
            X,
            y,
            self.oversampler,
            minority_label=minority_label,
            metric=self.metric,
            hidden_ratio=self.hidden_ratio,
            reference=self.reference,
            random_state=(
                self.random_state
                if isinstance(self.random_state, (int, type(None)))
                else None
            ),
            n_repeats=self.n_repeats,
        ),
        details=details,
    )
    self.minority_label_ = minority_label
    return self

score(X=None, y=None)

Return the negated error rate, so greater is better.

Scikit-learn's convention is that a higher score is better, but a higher error rate is worse. Returning the raw rate would make GridSearchCV select the worst sampler, so it is negated here.

Parameters:

Name	Type	Description	Default
X	NDArray[floating] | None	Optional data to validate instead of the fitted run.	None
y	NDArray[integer] | None	Labels matching X.	None

Returns:

Type	Description
float	Negated error rate.

Source code in src/oversampleqa/estimator.py

def score(
    self,
    X: NDArray[np.floating] | None = None,
    y: NDArray[np.integer] | None = None,
) -> float:
    """Return the negated error rate, so greater is better.

    Scikit-learn's convention is that a higher score is better, but a
    higher error rate is worse. Returning the raw rate would make
    ``GridSearchCV`` select the *worst* sampler, so it is negated here.

    Args:
        X: Optional data to validate instead of the fitted run.
        y: Labels matching ``X``.

    Returns:
        Negated error rate.
    """
    if X is not None and y is not None:
        # deep=False: nested `oversampler__*` keys are not constructor
        # arguments, which is why sklearn's own clone() uses shallow params.
        fresh = self.__class__(**self.get_params(deep=False))
        return -float(fresh.fit(X, y).error_rate_)
    if not hasattr(self, "error_rate_"):
        raise ValidationError("call fit before score, or pass X and y")
    return -self.error_rate_

BoundaryReport dataclass

How often synthetic points land in majority territory.

Source code in src/oversampleqa/fidelity.py

@dataclass(frozen=True)
class BoundaryReport:
    """How often synthetic points land in majority territory."""

    strict_rate: float
    graded_rate: float
    k: int
    metric: str
    n_synthetic: int

    def to_dict(self) -> dict[str, Any]:
        """Flat mapping for the reporting layer."""
        return {
            "boundary_violation_strict": self.strict_rate,
            "boundary_violation_graded": self.graded_rate,
            "boundary_k": self.k,
            "metric": self.metric,
            "n_synthetic": self.n_synthetic,
        }

to_dict()

Flat mapping for the reporting layer.

Source code in src/oversampleqa/fidelity.py

def to_dict(self) -> dict[str, Any]:
    """Flat mapping for the reporting layer."""
    return {
        "boundary_violation_strict": self.strict_rate,
        "boundary_violation_graded": self.graded_rate,
        "boundary_k": self.k,
        "metric": self.metric,
        "n_synthetic": self.n_synthetic,
    }

FidelityReport dataclass

Every fidelity signal for one oversampler on one dataset.

Source code in src/oversampleqa/fidelity.py

@dataclass(frozen=True)
class FidelityReport:
    """Every fidelity signal for one oversampler on one dataset."""

    error_rate: float
    manifold: ManifoldMetrics
    memorisation: MemorisationReport
    boundary: BoundaryReport
    utility: UtilityReport | None = None

    def to_dict(self) -> dict[str, Any]:
        """Flat mapping across every component."""
        payload: dict[str, Any] = {"error_rate": self.error_rate}
        payload.update(self.manifold.to_dict())
        payload.update(self.memorisation.to_dict())
        payload.update(self.boundary.to_dict())
        if self.utility is not None:
            payload.update(self.utility.to_dict())
        return payload

    def to_frame(self) -> pd.DataFrame:
        """Single-row frame, for concatenating across samplers."""
        return pd.DataFrame([self.to_dict()])

    def interpret(self) -> list[str]:
        """Readings of the patterns that matter, in plain language."""
        notes: list[str] = []
        if self.memorisation.distance_ratio < 0.1:
            notes.append(
                "Memorisation: synthetic points sit on top of training points. "
                "The error rate cannot say anything about synthesis quality here."
            )
        if self.manifold.coverage < 0.5:
            notes.append(
                f"Low coverage ({self.manifold.coverage:.2f}): the generator misses "
                "much of the real minority distribution."
            )
        if self.manifold.precision < 0.5:
            notes.append(
                f"Low precision ({self.manifold.precision:.2f}): many synthetic "
                "points fall outside the real manifold."
            )
        if self.boundary.strict_rate > 0.1:
            notes.append(
                f"Boundary violations ({self.boundary.strict_rate:.2f}): synthetic "
                "points are landing in majority territory."
            )
        if self.utility is not None and not self.utility.helps:
            notes.append("No downstream gain: the improvement interval includes zero.")
        if not notes:
            notes.append("No fidelity concerns detected.")
        return notes

to_dict()

Flat mapping across every component.

Source code in src/oversampleqa/fidelity.py

def to_dict(self) -> dict[str, Any]:
    """Flat mapping across every component."""
    payload: dict[str, Any] = {"error_rate": self.error_rate}
    payload.update(self.manifold.to_dict())
    payload.update(self.memorisation.to_dict())
    payload.update(self.boundary.to_dict())
    if self.utility is not None:
        payload.update(self.utility.to_dict())
    return payload

to_frame()

Single-row frame, for concatenating across samplers.

Source code in src/oversampleqa/fidelity.py

def to_frame(self) -> pd.DataFrame:
    """Single-row frame, for concatenating across samplers."""
    return pd.DataFrame([self.to_dict()])

interpret()

Readings of the patterns that matter, in plain language.

Source code in src/oversampleqa/fidelity.py

def interpret(self) -> list[str]:
    """Readings of the patterns that matter, in plain language."""
    notes: list[str] = []
    if self.memorisation.distance_ratio < 0.1:
        notes.append(
            "Memorisation: synthetic points sit on top of training points. "
            "The error rate cannot say anything about synthesis quality here."
        )
    if self.manifold.coverage < 0.5:
        notes.append(
            f"Low coverage ({self.manifold.coverage:.2f}): the generator misses "
            "much of the real minority distribution."
        )
    if self.manifold.precision < 0.5:
        notes.append(
            f"Low precision ({self.manifold.precision:.2f}): many synthetic "
            "points fall outside the real manifold."
        )
    if self.boundary.strict_rate > 0.1:
        notes.append(
            f"Boundary violations ({self.boundary.strict_rate:.2f}): synthetic "
            "points are landing in majority territory."
        )
    if self.utility is not None and not self.utility.helps:
        notes.append("No downstream gain: the improvement interval includes zero.")
    if not notes:
        notes.append("No fidelity concerns detected.")
    return notes

ManifoldMetrics dataclass

k-NN manifold estimates of fidelity and diversity.

Attributes

precision: Fraction of synthetic points inside the real manifold. Fidelity: are the generated points plausible? recall: Fraction of real points inside the synthetic manifold. Diversity: does the generator cover the real distribution? density: Like precision, but counts how many real k-NN spheres contain each synthetic point. Not saturated by a single real outlier whose sphere is enormous, which is precision's main failure mode. coverage: Fraction of real points with at least one synthetic point inside their own k-NN sphere. More robust than recall for the same reason.

Notes

Density and coverage are the more reliable pair (Naeem et al. 2020) and are what the report surfaces first. Precision and recall are reported too, because their disagreement with density/coverage is itself informative: it usually means an outlier is inflating one manifold.

Source code in src/oversampleqa/fidelity.py

@dataclass(frozen=True)
class ManifoldMetrics:
    """k-NN manifold estimates of fidelity and diversity.

    Attributes
    ----------
    precision:
        Fraction of synthetic points inside the real manifold. **Fidelity**:
        are the generated points plausible?
    recall:
        Fraction of real points inside the synthetic manifold. **Diversity**:
        does the generator cover the real distribution?
    density:
        Like precision, but counts *how many* real k-NN spheres contain each
        synthetic point. Not saturated by a single real outlier whose sphere is
        enormous, which is precision's main failure mode.
    coverage:
        Fraction of real points with at least one synthetic point inside their
        own k-NN sphere. More robust than recall for the same reason.

    Notes
    -----
    **Density and coverage are the more reliable pair** (Naeem et al. 2020) and
    are what the report surfaces first. Precision and recall are reported too,
    because their disagreement with density/coverage is itself informative: it
    usually means an outlier is inflating one manifold.
    """

    precision: float
    recall: float
    density: float
    coverage: float
    k: int
    metric: str
    n_synthetic: int
    n_real: int

    def to_dict(self) -> dict[str, Any]:
        """Flat mapping for the reporting layer."""
        return {
            "precision": self.precision,
            "recall": self.recall,
            "density": self.density,
            "coverage": self.coverage,
            "k": self.k,
            "metric": self.metric,
            "n_synthetic": self.n_synthetic,
            "n_real": self.n_real,
        }

to_dict()

Flat mapping for the reporting layer.

Source code in src/oversampleqa/fidelity.py

def to_dict(self) -> dict[str, Any]:
    """Flat mapping for the reporting layer."""
    return {
        "precision": self.precision,
        "recall": self.recall,
        "density": self.density,
        "coverage": self.coverage,
        "k": self.k,
        "metric": self.metric,
        "n_synthetic": self.n_synthetic,
        "n_real": self.n_real,
    }

MemorisationReport dataclass

How much of the "synthetic" output is really copied training data.

Attributes

distance_ratio: The headline number. Median nearest-neighbour distance from synthetic points to their training set, divided by the median nearest-neighbour distance within the real minority. Below 1 means the generator sits closer to its training points than real points sit to each other -- it is copying. Near 0 means outright duplication. exact_duplicate_rate: Fraction of synthetic points exactly coinciding with a training point. near_duplicate_rates: Fraction within a threshold taken from the real minority's own nearest-neighbour distance distribution, keyed by quantile. Deriving the threshold from the data makes it scale-free: an absolute tolerance means something different on every dataset.

Source code in src/oversampleqa/fidelity.py

@dataclass(frozen=True)
class MemorisationReport:
    """How much of the "synthetic" output is really copied training data.

    Attributes
    ----------
    distance_ratio:
        **The headline number.** Median nearest-neighbour distance from
        synthetic points to their training set, divided by the median
        nearest-neighbour distance *within* the real minority. Below 1 means
        the generator sits closer to its training points than real points sit
        to each other -- it is copying. Near 0 means outright duplication.
    exact_duplicate_rate:
        Fraction of synthetic points exactly coinciding with a training point.
    near_duplicate_rates:
        Fraction within a threshold taken from the real minority's own
        nearest-neighbour distance distribution, keyed by quantile. Deriving
        the threshold from the data makes it scale-free: an absolute tolerance
        means something different on every dataset.
    """

    distance_ratio: float
    exact_duplicate_rate: float
    near_duplicate_rates: dict[float, float]
    median_distance_to_train: float
    median_real_nn_distance: float
    metric: str
    n_synthetic: int

    def to_dict(self) -> dict[str, Any]:
        """Flat mapping for the reporting layer."""
        payload: dict[str, Any] = {
            "memorisation_distance_ratio": self.distance_ratio,
            "exact_duplicate_rate": self.exact_duplicate_rate,
            "median_distance_to_train": self.median_distance_to_train,
            "median_real_nn_distance": self.median_real_nn_distance,
            "metric": self.metric,
            "n_synthetic": self.n_synthetic,
        }
        for quantile, rate in self.near_duplicate_rates.items():
            payload[f"near_duplicate_rate_q{quantile:g}"] = rate
        return payload

    def interpret(self) -> str:
        """One-line reading of the headline ratio."""
        if np.isnan(self.distance_ratio):
            return "Not enough data to assess memorisation."
        if self.distance_ratio < 0.1:
            return (
                f"ratio {self.distance_ratio:.3f}: synthetic points sit essentially "
                "on top of training points -- this generator is copying."
            )
        if self.distance_ratio < 0.5:
            return (
                f"ratio {self.distance_ratio:.3f}: synthetic points are much closer "
                "to training data than real points are to each other."
            )
        return (
            f"ratio {self.distance_ratio:.3f}: synthetic points are about as far "
            "from training data as real points are from each other."
        )

to_dict()

Flat mapping for the reporting layer.

Source code in src/oversampleqa/fidelity.py

def to_dict(self) -> dict[str, Any]:
    """Flat mapping for the reporting layer."""
    payload: dict[str, Any] = {
        "memorisation_distance_ratio": self.distance_ratio,
        "exact_duplicate_rate": self.exact_duplicate_rate,
        "median_distance_to_train": self.median_distance_to_train,
        "median_real_nn_distance": self.median_real_nn_distance,
        "metric": self.metric,
        "n_synthetic": self.n_synthetic,
    }
    for quantile, rate in self.near_duplicate_rates.items():
        payload[f"near_duplicate_rate_q{quantile:g}"] = rate
    return payload

interpret()

One-line reading of the headline ratio.

Source code in src/oversampleqa/fidelity.py

def interpret(self) -> str:
    """One-line reading of the headline ratio."""
    if np.isnan(self.distance_ratio):
        return "Not enough data to assess memorisation."
    if self.distance_ratio < 0.1:
        return (
            f"ratio {self.distance_ratio:.3f}: synthetic points sit essentially "
            "on top of training points -- this generator is copying."
        )
    if self.distance_ratio < 0.5:
        return (
            f"ratio {self.distance_ratio:.3f}: synthetic points are much closer "
            "to training data than real points are to each other."
        )
    return (
        f"ratio {self.distance_ratio:.3f}: synthetic points are about as far "
        "from training data as real points are from each other."
    )

NullCalibration dataclass

Where an observed error rate sits against known reference points.

Attributes

observed: The error rate being interpreted. null_rates: Error rates from scoring real held-out minority points -- what an ideal generator, drawing from the true minority distribution, achieves. ceiling_rates: Error rates from deliberately bad points drawn from the majority region. The other end of the scale. z_score: (observed - null_mean) / null_sd. Positive means worse than ideal. nan when the null has no spread. percentile: Empirical percentile of observed within null_rates. scaled: Position on a 0-1 scale where 0 is the null mean and 1 the ceiling mean. Above 1 is worse than a deliberately bad generator.

Source code in src/oversampleqa/inference.py

@dataclass(frozen=True)
class NullCalibration:
    """Where an observed error rate sits against known reference points.

    Attributes
    ----------
    observed:
        The error rate being interpreted.
    null_rates:
        Error rates from scoring *real* held-out minority points -- what an
        ideal generator, drawing from the true minority distribution, achieves.
    ceiling_rates:
        Error rates from deliberately bad points drawn from the majority
        region. The other end of the scale.
    z_score:
        ``(observed - null_mean) / null_sd``. Positive means worse than ideal.
        ``nan`` when the null has no spread.
    percentile:
        Empirical percentile of ``observed`` within ``null_rates``.
    scaled:
        Position on a 0-1 scale where 0 is the null mean and 1 the ceiling
        mean. Above 1 is worse than a deliberately bad generator.
    """

    observed: float
    null_rates: tuple[float, ...]
    ceiling_rates: tuple[float, ...]
    z_score: float
    percentile: float
    scaled: float
    metric: str
    n_draws: int

    @property
    def null_mean(self) -> float:
        """Mean of the null distribution."""
        return float(np.mean(self.null_rates)) if self.null_rates else float("nan")

    @property
    def null_sd(self) -> float:
        """Standard deviation of the null distribution."""
        if len(self.null_rates) < 2:
            return float("nan")
        return float(np.std(self.null_rates, ddof=1))

    @property
    def ceiling_mean(self) -> float:
        """Mean of the ceiling distribution."""
        return (
            float(np.mean(self.ceiling_rates)) if self.ceiling_rates else float("nan")
        )

    def null_interval(self, confidence: float = 0.95) -> tuple[float, float]:
        """Percentile interval of the null distribution."""
        if len(self.null_rates) < 2:
            return (float("nan"), float("nan"))
        alpha = 1.0 - confidence
        arr = np.asarray(self.null_rates)
        return (
            float(np.percentile(arr, 100 * alpha / 2)),
            float(np.percentile(arr, 100 * (1 - alpha / 2))),
        )

    def interpret(self) -> str:
        """One-line reading of where the observed rate falls."""
        low, high = self.null_interval()
        if np.isnan(low):
            return "Not enough draws to calibrate."
        if self.observed < low:
            # Previously reported as "within". Below the interval is a distinct
            # and more interesting outcome than inside it: real held-out
            # minority points score in [low, high], so beating that is not
            # "better synthesis" -- points closer to the minority than real
            # minority points are usually sitting on top of the training data.
            return (
                f"{self.observed:.3f} is below the null interval "
                f"[{low:.3f}, {high:.3f}] (z={self.z_score:.2f}) -- better than "
                "real held-out minority points score. Check memorisation before "
                "reading this as quality."
            )
        if self.observed <= high:
            return (
                f"{self.observed:.3f} is within the null interval "
                f"[{low:.3f}, {high:.3f}] -- indistinguishable from an ideal "
                "generator on this data."
            )
        return (
            f"{self.observed:.3f} is above the null interval "
            f"[{low:.3f}, {high:.3f}] (z={self.z_score:.2f}) -- worse than an "
            "ideal generator would achieve here."
        )

    def to_dict(self) -> dict[str, Any]:
        """Flat mapping for the reporting layer."""
        low, high = self.null_interval()
        return {
            "observed": self.observed,
            "null_mean": self.null_mean,
            "null_sd": self.null_sd,
            "null_ci_lower": low,
            "null_ci_upper": high,
            "ceiling_mean": self.ceiling_mean,
            "z_score": self.z_score,
            "percentile": self.percentile,
            "scaled": self.scaled,
            "metric": self.metric,
            "n_draws": self.n_draws,
        }

null_mean property

Mean of the null distribution.

null_sd property

Standard deviation of the null distribution.

ceiling_mean property

Mean of the ceiling distribution.

null_interval(confidence=0.95)

Percentile interval of the null distribution.

Source code in src/oversampleqa/inference.py

def null_interval(self, confidence: float = 0.95) -> tuple[float, float]:
    """Percentile interval of the null distribution."""
    if len(self.null_rates) < 2:
        return (float("nan"), float("nan"))
    alpha = 1.0 - confidence
    arr = np.asarray(self.null_rates)
    return (
        float(np.percentile(arr, 100 * alpha / 2)),
        float(np.percentile(arr, 100 * (1 - alpha / 2))),
    )

interpret()

One-line reading of where the observed rate falls.

Source code in src/oversampleqa/inference.py

def interpret(self) -> str:
    """One-line reading of where the observed rate falls."""
    low, high = self.null_interval()
    if np.isnan(low):
        return "Not enough draws to calibrate."
    if self.observed < low:
        # Previously reported as "within". Below the interval is a distinct
        # and more interesting outcome than inside it: real held-out
        # minority points score in [low, high], so beating that is not
        # "better synthesis" -- points closer to the minority than real
        # minority points are usually sitting on top of the training data.
        return (
            f"{self.observed:.3f} is below the null interval "
            f"[{low:.3f}, {high:.3f}] (z={self.z_score:.2f}) -- better than "
            "real held-out minority points score. Check memorisation before "
            "reading this as quality."
        )
    if self.observed <= high:
        return (
            f"{self.observed:.3f} is within the null interval "
            f"[{low:.3f}, {high:.3f}] -- indistinguishable from an ideal "
            "generator on this data."
        )
    return (
        f"{self.observed:.3f} is above the null interval "
        f"[{low:.3f}, {high:.3f}] (z={self.z_score:.2f}) -- worse than an "
        "ideal generator would achieve here."
    )

to_dict()

Flat mapping for the reporting layer.

Source code in src/oversampleqa/inference.py

def to_dict(self) -> dict[str, Any]:
    """Flat mapping for the reporting layer."""
    low, high = self.null_interval()
    return {
        "observed": self.observed,
        "null_mean": self.null_mean,
        "null_sd": self.null_sd,
        "null_ci_lower": low,
        "null_ci_upper": high,
        "ceiling_mean": self.ceiling_mean,
        "z_score": self.z_score,
        "percentile": self.percentile,
        "scaled": self.scaled,
        "metric": self.metric,
        "n_draws": self.n_draws,
    }

TwoSampleTestResult dataclass

Outcome of a two-sample test between synthetic and real points.

A high p-value is weak evidence that the two samples are distributionally indistinguishable -- which is what good synthesis looks like. See the warning in :func:nn_two_sample_test about what failing to reject does not mean.

Source code in src/oversampleqa/inference.py

@dataclass(frozen=True)
class TwoSampleTestResult:
    """Outcome of a two-sample test between synthetic and real points.

    A **high** p-value is weak evidence that the two samples are
    distributionally indistinguishable -- which is what good synthesis looks
    like. See the warning in :func:`nn_two_sample_test` about what failing to
    reject does *not* mean.
    """

    name: str
    statistic: float
    p_value: float
    n_synthetic: int
    n_real: int
    n_permutations: int
    asymptotic_p_value: float | None = None
    null_statistics: tuple[float, ...] = field(default=(), repr=False)

    def to_dict(self) -> dict[str, Any]:
        """Flat mapping for the reporting layer."""
        return {
            "test": self.name,
            "statistic": self.statistic,
            "p_value": self.p_value,
            "asymptotic_p_value": self.asymptotic_p_value,
            "n_synthetic": self.n_synthetic,
            "n_real": self.n_real,
            "n_permutations": self.n_permutations,
        }

to_dict()

Flat mapping for the reporting layer.

Source code in src/oversampleqa/inference.py

def to_dict(self) -> dict[str, Any]:
    """Flat mapping for the reporting layer."""
    return {
        "test": self.name,
        "statistic": self.statistic,
        "p_value": self.p_value,
        "asymptotic_p_value": self.asymptotic_p_value,
        "n_synthetic": self.n_synthetic,
        "n_real": self.n_real,
        "n_permutations": self.n_permutations,
    }

MemoryEfficientValidator

Drop-in replacement for :func:validate_oversampling with memory safeguards.

Source code in src/oversampleqa/memory_efficient_validator.py

class MemoryEfficientValidator:
    """Drop-in replacement for :func:`validate_oversampling` with memory safeguards."""

    def __init__(
        self,
        memory_limit_gb: float = 4.0,
        batch_size: int | str = "auto",
        show_progress: bool = False,
        temp_dir: str | None = None,
        cache: ValidationCache | None = None,
    ) -> None:
        self.memory_limit_gb = memory_limit_gb
        self.batch_size = batch_size
        self.temp_root = (
            Path(temp_dir) if temp_dir else Path(tempfile.gettempdir()) / "oversampleqa"
        )
        self.temp_root.mkdir(parents=True, exist_ok=True)
        self.cache = cache or ValidationCache()
        self.distance_computer = OptimizedDistanceMatrix(
            memory_limit_gb=memory_limit_gb,
            metric_registry=_METRICS,
            show_progress=show_progress,
            cache=self.cache,
        )
        self._stream_dirs: list[Path] = []

    def validate_oversampling(
        self,
        X: NDArray[np.floating],
        y: NDArray[np.integer],
        minority_label: int,
        oversampler: BaseOverSampler,
        hidden_ratio: float = 0.1,
        metric: str = "hassanat",
        metric_kwargs: dict[str, Any] | None = None,
        return_details: bool = False,
        *,
        reference: ReferenceSet = "hidden_minority",
        minority_hidden_ratio: float | None = None,
        min_hidden: int = 5,
        random_state: RandomStateLike = 42,
        stratify_by: NDArray[Any] | None = None,
    ) -> float | ValidationDetails:
        """Validate oversampling with streaming-aware distance calculations.

        Uses the same estimand as :func:`oversampleqa.validate_oversampling`
        via the shared :func:`~oversampleqa.validator.prepare_validation_split`
        helper, so the two cannot drift apart.

        Args:
            X: Feature matrix.
            y: Target labels.
            minority_label: Minority class label.
            oversampler: Oversampler instance.
            hidden_ratio: Fraction of majority to hide.
            metric: Distance metric name.
            metric_kwargs: Metric keyword arguments.
            return_details: Whether to return a ``ValidationDetails``.
            reference: Which minority set to compare against. See
                :func:`oversampleqa.validate_oversampling`.
            minority_hidden_ratio: Fraction of the minority to hide.
            min_hidden: Minimum held-out minority points.

        Returns:
            Error rate, or ``ValidationDetails`` when ``return_details`` is True.
        """
        X = np.asarray(X)
        y = np.asarray(y)
        metric_kwargs = metric_kwargs or {}
        warn_reference_bias(reference, stacklevel=3)

        params_hash = None
        if self.cache is not None and not return_details:
            payload = {
                "data_hash": self.cache.get_data_hash(X, y),
                "minority_label": minority_label,
                "oversampler": oversampler.__class__.__qualname__,
                "oversampler_params": oversampler.get_params(deep=True),
                "hidden_ratio": hidden_ratio,
                "metric": metric,
                "metric_kwargs": metric_kwargs,
                "reference": reference,
                "minority_hidden_ratio": minority_hidden_ratio,
                "random_state": random_state
                if isinstance(random_state, (int, type(None)))
                else "generator",
            }
            params_hash = hashlib.sha256(pickle.dumps(payload)).hexdigest()
            cached = self.cache.load_validation_result(params_hash)
            if cached is not None:
                return cached

        labels = np.unique(y)
        majority_labels = labels[labels != minority_label]
        if len(majority_labels) == 0:
            raise ValueError(f"minority_label {minority_label} is the only label in y")
        majority_label = int(majority_labels[0])

        split = prepare_validation_split(
            X,
            y,
            minority_label,
            majority_label,
            hidden_ratio,
            reference=reference,
            minority_hidden_ratio=minority_hidden_ratio,
            min_hidden=min_hidden,
            random_state=random_state,
            stratify_by=stratify_by,
        )
        X_train = split.X_train
        y_train = split.y_train
        hid_majority = split.hid_majority
        fit_minority = split.fit_minority
        minority = split.reference_minority

        X_res, y_res = oversampler.fit_resample(X_train, y_train)
        synthetic = extract_synthetic_samples(X_train, X_res, y_res, minority_label)

        empty = np.empty((0, 0))
        if len(synthetic) == 0:
            warnings.warn(
                f"{type(oversampler).__name__} produced no synthetic minority "
                "samples, so there is nothing to validate. Returning nan rather "
                "than 0.0, which would be indistinguishable from a perfect score.",
                UserWarning,
                stacklevel=2,
            )
            if return_details:
                return ValidationDetails(
                    error_rate=float("nan"),
                    n_errors=0,
                    n_synthetic=0,
                    n_ties=0,
                    duplication_rate=float("nan"),
                    reference=reference,
                    dist_hidden=empty,
                    dist_min=empty,
                )
            return float("nan")

        dtype = np.result_type(synthetic.dtype, minority.dtype, np.float64)
        est_hidden = self.distance_computer.estimate_memory_gb(
            len(synthetic), len(hid_majority), dtype=dtype
        )
        est_minority = self.distance_computer.estimate_memory_gb(
            len(synthetic), len(minority), dtype=dtype
        )
        available = get_available_memory_gb()
        requires_stream = max(len(synthetic), len(minority), len(hid_majority)) > 10_000
        if est_hidden > self.memory_limit_gb or est_minority > self.memory_limit_gb:
            warnings.warn(
                "Distance matrices exceed configured memory limit; activating streaming mode.",
                ResourceWarning,
                stacklevel=2,
            )
            requires_stream = True
        elif est_hidden > available or est_minority > available:
            warnings.warn(
                "Estimated distance matrices exceed available system memory; switching to streaming mode.",
                ResourceWarning,
                stacklevel=2,
            )
            requires_stream = True

        if requires_stream:
            return self._streaming_validation(
                synthetic,
                hid_majority,
                minority,
                metric=metric,
                metric_kwargs=metric_kwargs,
                return_details=return_details,
                reference=reference,
                fit_minority=fit_minority,
            )

        dist_hidden = self.distance_computer.compute_distance_matrix(
            synthetic,
            hid_majority,
            metric=metric,
            batch_size=self.batch_size,
            **metric_kwargs,
        )
        dist_min = self.distance_computer.compute_distance_matrix(
            synthetic,
            minority,
            metric=metric,
            batch_size=self.batch_size,
            **metric_kwargs,
        )

        nearest_hidden = (
            dist_hidden.min(axis=1)
            if dist_hidden.size
            else np.full(len(synthetic), np.inf)
        )
        nearest_min = (
            dist_min.min(axis=1) if dist_min.size else np.full(len(synthetic), np.inf)
        )
        errors, n_ties = score_nearest_distances(nearest_hidden, nearest_min)
        rate = calculate_error_rate(errors, len(synthetic))

        if return_details:
            return ValidationDetails(
                error_rate=rate,
                n_errors=errors,
                n_synthetic=len(synthetic),
                n_ties=n_ties,
                duplication_rate=duplication_rate(synthetic, fit_minority),
                reference=reference,
                dist_hidden=dist_hidden,
                dist_min=dist_min,
            )

        if params_hash is not None:
            self.cache.cache_validation_result(params_hash, rate)
        return rate

    def _streaming_validation(
        self,
        synthetic: NDArray[np.floating],
        hidden_majority: NDArray[np.floating],
        minority: NDArray[np.floating],
        metric: str,
        metric_kwargs: dict[str, Any],
        return_details: bool,
        reference: ReferenceSet = "hidden_minority",
        fit_minority: NDArray[np.floating] | None = None,
    ) -> float | ValidationDetails:
        """Compute validation statistics using chunked distance matrices.

        Args:
            synthetic: Synthetic samples.
            hidden_majority: Hidden majority samples.
            minority: Minority reference samples.
            metric: Distance metric name.
            metric_kwargs: Metric keyword arguments.
            return_details: Whether to return distance matrices.
            reference: Which minority set was used.
            fit_minority: Minority points the oversampler trained on, for the
                duplication diagnostic.

        Returns:
            Error rate, or ``ValidationDetails`` when ``return_details`` is True.
        """
        dtype = np.result_type(synthetic.dtype, minority.dtype, np.float64)
        n_syn = len(synthetic)
        n_hidden = len(hidden_majority)
        n_minority = len(minority)

        chunk_cols = max(1, n_hidden, n_minority)
        chunk_size = min(n_syn, self._stream_chunk_size(chunk_cols, dtype=dtype))
        errors = 0
        n_ties = 0

        hidden_store: NDArray[np.floating] | None
        min_store: NDArray[np.floating] | None
        hidden_store = None
        min_store = None

        if return_details:
            temp_dir = Path(
                tempfile.mkdtemp(prefix="oversampleqa_stream_", dir=self.temp_root)
            )
            self._stream_dirs.append(temp_dir)
            if n_hidden > 0:
                hidden_store = np.memmap(
                    temp_dir / "hidden.dat",
                    dtype=dtype,
                    mode="w+",
                    shape=(n_syn, n_hidden),
                )
            else:
                hidden_store = np.empty((n_syn, 0), dtype=dtype)
            if n_minority > 0:
                min_store = np.memmap(
                    temp_dir / "minority.dat",
                    dtype=dtype,
                    mode="w+",
                    shape=(n_syn, n_minority),
                )
            else:
                min_store = np.empty((n_syn, 0), dtype=dtype)

        for start in range(0, n_syn, chunk_size):
            end = min(start + chunk_size, n_syn)
            chunk = synthetic[start:end]

            if n_hidden > 0:
                dist_hidden = self.distance_computer.compute_distance_matrix(
                    chunk,
                    hidden_majority,
                    metric=metric,
                    batch_size=self.batch_size,
                    **metric_kwargs,
                )
                if return_details and hidden_store is not None:
                    hidden_store[start:end] = dist_hidden
                nearest_hidden = dist_hidden.min(axis=1)
            else:
                nearest_hidden = np.full(len(chunk), np.inf)

            if n_minority > 0:
                dist_min = self.distance_computer.compute_distance_matrix(
                    chunk,
                    minority,
                    metric=metric,
                    batch_size=self.batch_size,
                    **metric_kwargs,
                )
                if return_details and min_store is not None:
                    min_store[start:end] = dist_min
                nearest_min = dist_min.min(axis=1)
            else:
                nearest_min = np.full(len(chunk), np.inf)

            chunk_errors, chunk_ties = score_nearest_distances(
                nearest_hidden, nearest_min
            )
            errors += chunk_errors
            n_ties += chunk_ties

        rate = calculate_error_rate(errors, n_syn)

        if return_details:
            if isinstance(hidden_store, np.memmap):
                hidden_store.flush()
            if isinstance(min_store, np.memmap):
                min_store.flush()
            hidden_return = (
                hidden_store
                if hidden_store is not None
                else np.empty((n_syn, 0), dtype=dtype)
            )
            min_return = (
                min_store
                if min_store is not None
                else np.empty((n_syn, 0), dtype=dtype)
            )
            dup = (
                duplication_rate(synthetic, fit_minority)
                if fit_minority is not None
                else float("nan")
            )
            return ValidationDetails(
                error_rate=rate,
                n_errors=errors,
                n_synthetic=n_syn,
                n_ties=n_ties,
                duplication_rate=dup,
                reference=reference,
                dist_hidden=hidden_return,
                dist_min=min_return,
            )

        return rate

    def _stream_chunk_size(self, n_cols: int, dtype: np.dtype[Any]) -> int:
        """Return streaming chunk size based on memory limit.

        Args:
            n_cols: Number of columns in distance matrix.
            dtype: Data type of the distance matrix.

        Returns:
            Maximum number of rows to process per chunk.
        """
        limit_bytes = int(self.memory_limit_gb * (1024**3))
        per_row = max(1, n_cols * np.dtype(dtype).itemsize)
        return max(1, limit_bytes // per_row)

    def cleanup(self) -> None:
        """Remove temporary files created during streaming computations.

        This cleans any memmap-backed temporary directories created during
        streaming validation.
        """
        for path in self._stream_dirs:
            shutil.rmtree(path, ignore_errors=True)
        self._stream_dirs.clear()

validate_oversampling(X, y, minority_label, oversampler, hidden_ratio=0.1, metric='hassanat', metric_kwargs=None, return_details=False, *, reference='hidden_minority', minority_hidden_ratio=None, min_hidden=5, random_state=42, stratify_by=None)

Validate oversampling with streaming-aware distance calculations.

Uses the same estimand as :func:oversampleqa.validate_oversampling via the shared :func:~oversampleqa.validator.prepare_validation_split helper, so the two cannot drift apart.

Parameters:

Name	Type	Description	Default
X	NDArray[floating]	Feature matrix.	required
y	NDArray[integer]	Target labels.	required
minority_label	int	Minority class label.	required
oversampler	BaseOverSampler	Oversampler instance.	required
hidden_ratio	float	Fraction of majority to hide.	0.1
metric	str	Distance metric name.	'hassanat'
metric_kwargs	dict[str, Any] | None	Metric keyword arguments.	None
return_details	bool	Whether to return a ValidationDetails.	False
reference	ReferenceSet	Which minority set to compare against. See :func:oversampleqa.validate_oversampling.	'hidden_minority'
minority_hidden_ratio	float | None	Fraction of the minority to hide.	None
min_hidden	int	Minimum held-out minority points.	5

Returns:

Type	Description
float | ValidationDetails	Error rate, or ValidationDetails when return_details is True.

Source code in src/oversampleqa/memory_efficient_validator.py

def validate_oversampling(
    self,
    X: NDArray[np.floating],
    y: NDArray[np.integer],
    minority_label: int,
    oversampler: BaseOverSampler,
    hidden_ratio: float = 0.1,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
    return_details: bool = False,
    *,
    reference: ReferenceSet = "hidden_minority",
    minority_hidden_ratio: float | None = None,
    min_hidden: int = 5,
    random_state: RandomStateLike = 42,
    stratify_by: NDArray[Any] | None = None,
) -> float | ValidationDetails:
    """Validate oversampling with streaming-aware distance calculations.

    Uses the same estimand as :func:`oversampleqa.validate_oversampling`
    via the shared :func:`~oversampleqa.validator.prepare_validation_split`
    helper, so the two cannot drift apart.

    Args:
        X: Feature matrix.
        y: Target labels.
        minority_label: Minority class label.
        oversampler: Oversampler instance.
        hidden_ratio: Fraction of majority to hide.
        metric: Distance metric name.
        metric_kwargs: Metric keyword arguments.
        return_details: Whether to return a ``ValidationDetails``.
        reference: Which minority set to compare against. See
            :func:`oversampleqa.validate_oversampling`.
        minority_hidden_ratio: Fraction of the minority to hide.
        min_hidden: Minimum held-out minority points.

    Returns:
        Error rate, or ``ValidationDetails`` when ``return_details`` is True.
    """
    X = np.asarray(X)
    y = np.asarray(y)
    metric_kwargs = metric_kwargs or {}
    warn_reference_bias(reference, stacklevel=3)

    params_hash = None
    if self.cache is not None and not return_details:
        payload = {
            "data_hash": self.cache.get_data_hash(X, y),
            "minority_label": minority_label,
            "oversampler": oversampler.__class__.__qualname__,
            "oversampler_params": oversampler.get_params(deep=True),
            "hidden_ratio": hidden_ratio,
            "metric": metric,
            "metric_kwargs": metric_kwargs,
            "reference": reference,
            "minority_hidden_ratio": minority_hidden_ratio,
            "random_state": random_state
            if isinstance(random_state, (int, type(None)))
            else "generator",
        }
        params_hash = hashlib.sha256(pickle.dumps(payload)).hexdigest()
        cached = self.cache.load_validation_result(params_hash)
        if cached is not None:
            return cached

    labels = np.unique(y)
    majority_labels = labels[labels != minority_label]
    if len(majority_labels) == 0:
        raise ValueError(f"minority_label {minority_label} is the only label in y")
    majority_label = int(majority_labels[0])

    split = prepare_validation_split(
        X,
        y,
        minority_label,
        majority_label,
        hidden_ratio,
        reference=reference,
        minority_hidden_ratio=minority_hidden_ratio,
        min_hidden=min_hidden,
        random_state=random_state,
        stratify_by=stratify_by,
    )
    X_train = split.X_train
    y_train = split.y_train
    hid_majority = split.hid_majority
    fit_minority = split.fit_minority
    minority = split.reference_minority

    X_res, y_res = oversampler.fit_resample(X_train, y_train)
    synthetic = extract_synthetic_samples(X_train, X_res, y_res, minority_label)

    empty = np.empty((0, 0))
    if len(synthetic) == 0:
        warnings.warn(
            f"{type(oversampler).__name__} produced no synthetic minority "
            "samples, so there is nothing to validate. Returning nan rather "
            "than 0.0, which would be indistinguishable from a perfect score.",
            UserWarning,
            stacklevel=2,
        )
        if return_details:
            return ValidationDetails(
                error_rate=float("nan"),
                n_errors=0,
                n_synthetic=0,
                n_ties=0,
                duplication_rate=float("nan"),
                reference=reference,
                dist_hidden=empty,
                dist_min=empty,
            )
        return float("nan")

    dtype = np.result_type(synthetic.dtype, minority.dtype, np.float64)
    est_hidden = self.distance_computer.estimate_memory_gb(
        len(synthetic), len(hid_majority), dtype=dtype
    )
    est_minority = self.distance_computer.estimate_memory_gb(
        len(synthetic), len(minority), dtype=dtype
    )
    available = get_available_memory_gb()
    requires_stream = max(len(synthetic), len(minority), len(hid_majority)) > 10_000
    if est_hidden > self.memory_limit_gb or est_minority > self.memory_limit_gb:
        warnings.warn(
            "Distance matrices exceed configured memory limit; activating streaming mode.",
            ResourceWarning,
            stacklevel=2,
        )
        requires_stream = True
    elif est_hidden > available or est_minority > available:
        warnings.warn(
            "Estimated distance matrices exceed available system memory; switching to streaming mode.",
            ResourceWarning,
            stacklevel=2,
        )
        requires_stream = True

    if requires_stream:
        return self._streaming_validation(
            synthetic,
            hid_majority,
            minority,
            metric=metric,
            metric_kwargs=metric_kwargs,
            return_details=return_details,
            reference=reference,
            fit_minority=fit_minority,
        )

    dist_hidden = self.distance_computer.compute_distance_matrix(
        synthetic,
        hid_majority,
        metric=metric,
        batch_size=self.batch_size,
        **metric_kwargs,
    )
    dist_min = self.distance_computer.compute_distance_matrix(
        synthetic,
        minority,
        metric=metric,
        batch_size=self.batch_size,
        **metric_kwargs,
    )

    nearest_hidden = (
        dist_hidden.min(axis=1)
        if dist_hidden.size
        else np.full(len(synthetic), np.inf)
    )
    nearest_min = (
        dist_min.min(axis=1) if dist_min.size else np.full(len(synthetic), np.inf)
    )
    errors, n_ties = score_nearest_distances(nearest_hidden, nearest_min)
    rate = calculate_error_rate(errors, len(synthetic))

    if return_details:
        return ValidationDetails(
            error_rate=rate,
            n_errors=errors,
            n_synthetic=len(synthetic),
            n_ties=n_ties,
            duplication_rate=duplication_rate(synthetic, fit_minority),
            reference=reference,
            dist_hidden=dist_hidden,
            dist_min=dist_min,
        )

    if params_hash is not None:
        self.cache.cache_validation_result(params_hash, rate)
    return rate

cleanup()

Remove temporary files created during streaming computations.

This cleans any memmap-backed temporary directories created during streaming validation.

Source code in src/oversampleqa/memory_efficient_validator.py

def cleanup(self) -> None:
    """Remove temporary files created during streaming computations.

    This cleans any memmap-backed temporary directories created during
    streaming validation.
    """
    for path in self._stream_dirs:
        shutil.rmtree(path, ignore_errors=True)
    self._stream_dirs.clear()

OptimizedDistanceMatrix

Memory-aware distance matrix computation with vectorisation and batching.

.. note::

The effective memory limit is min(memory_limit_gb, available), where available comes from psutil. Without psutil installed it is assumed to be 1 GB, regardless of the machine, so batching is more conservative and throughput differs from an otherwise identical environment that has it. The fallback is logged once at INFO. Install the performance extra to get the real figure.

Parameters

cache_size : int, default=128 Retained for API compatibility. memory_limit_gb : float, default=4.0 Upper bound on the memory one computation may use. metric_registry : dict, optional Name-to-callable mapping of metrics. show_progress : bool, default=False Display a progress bar for large computations. progress_threshold : int, default=10000 Row count above which progress is shown. cache : ValidationCache, optional Opt-in cache. None means nothing is written to disk. safety_factor : float, default=0.8 Fraction of the limit a batched computation is allowed to plan against. The remainder is headroom for allocator overhead and transient copies, which the analytic estimate does not model.

Source code in src/oversampleqa/optimized_distance.py

class OptimizedDistanceMatrix:
    """Memory-aware distance matrix computation with vectorisation and batching.

    .. note::

       The effective memory limit is ``min(memory_limit_gb, available)``, where
       ``available`` comes from ``psutil``. **Without ``psutil`` installed it is
       assumed to be 1 GB**, regardless of the machine, so batching is more
       conservative and throughput differs from an otherwise identical
       environment that has it. The fallback is logged once at INFO. Install the
       ``performance`` extra to get the real figure.

    Parameters
    ----------
    cache_size : int, default=128
        Retained for API compatibility.
    memory_limit_gb : float, default=4.0
        Upper bound on the memory one computation may use.
    metric_registry : dict, optional
        Name-to-callable mapping of metrics.
    show_progress : bool, default=False
        Display a progress bar for large computations.
    progress_threshold : int, default=10000
        Row count above which progress is shown.
    cache : ValidationCache, optional
        Opt-in cache. ``None`` means nothing is written to disk.
    safety_factor : float, default=0.8
        Fraction of the limit a batched computation is allowed to plan against.
        The remainder is headroom for allocator overhead and transient copies,
        which the analytic estimate does not model.
    """

    def __init__(
        self,
        cache_size: int = 128,
        memory_limit_gb: float = 4.0,
        metric_registry: dict[str, DistanceCallable] | None = None,
        show_progress: bool = False,
        progress_threshold: int = 10_000,
        cache: ValidationCache | None = None,
        safety_factor: float = 0.8,
    ) -> None:
        if not 0.0 < safety_factor <= 1.0:
            raise ValueError(f"safety_factor must be in (0, 1]; got {safety_factor!r}")
        self.cache_size = cache_size
        self.memory_limit_gb = memory_limit_gb
        self.metric_registry = metric_registry or {}
        self.show_progress = show_progress
        self.progress_threshold = progress_threshold
        self.cache = cache
        self.safety_factor = safety_factor

        self._vectorized_dispatch: dict[str, Callable[..., NDArray[np.floating]]] = {
            "hassanat": self._vectorized_hassanat,
            "hamming": self._vectorized_hamming,
            "jaccard": self._vectorized_jaccard,
            "hellinger": self._vectorized_hellinger,
            "jensen_shannon": self._vectorized_jensen_shannon,
            "wasserstein": self._vectorized_wasserstein,
            # "energy" is deliberately absent. It is a sample-based metric whose
            # scalar form computes three pairwise-norm terms per (i, j) -- the
            # cross term plus a within-term for each input row. Broadcasting that
            # needs an (n1, n2, d, d) intermediate, which is larger than the work
            # it saves at any realistic size, so it stays on _pairwise.
            #
            "euclidean": self._vectorized_euclidean,
            "manhattan": self._vectorized_manhattan,
            "cosine": self._vectorized_cosine,
            "chebyshev": self._vectorized_chebyshev,
            "canberra": self._vectorized_canberra,
            "braycurtis": self._vectorized_braycurtis,
            "correlation": self._vectorized_correlation,
            "minkowski": self._vectorized_minkowski,
            "mahalanobis": self._vectorized_mahalanobis,
        }

    def compute_distance_matrix(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        metric: str = "hassanat",
        batch_size: int | str = "auto",
        **kwargs: Any,
    ) -> NDArray[np.floating]:
        """Compute pairwise distances with automatic optimisation.

        Parameters
        ----------
        X1, X2:
            Input matrices of shape ``(n_samples, n_features)``.
        metric:
            Name of the distance metric registered in ``metric_registry``.
        batch_size:
            ``"auto"`` selects the largest batch size fitting within
            ``memory_limit_gb``. An integer enforces a specific chunk length.
            ``"stream"`` yields rows sequentially without storing the full
            matrix in memory.
        kwargs:
            Extra keyword arguments forwarded to the underlying metric.
        """
        if metric not in self.metric_registry:
            raise ValueError(f"Unsupported metric '{metric}'")

        X1 = np.asarray(X1, dtype=float)
        X2 = np.asarray(X2, dtype=float)
        n1, n2 = X1.shape[0], X2.shape[0]
        dtype = np.result_type(X1.dtype, X2.dtype, np.float64)
        X1 = X1.astype(dtype, copy=False)
        X2 = X2.astype(dtype, copy=False)

        if n1 == 0 or n2 == 0:
            return np.empty((n1, n2), dtype=dtype)

        if isinstance(batch_size, str):
            batch_key = batch_size.lower()
        else:
            batch_key = ""

        if self.cache is not None and batch_key != "stream":
            return self.cache.cached_distance_matrix(
                optimizer=self,
                X1=X1,
                X2=X2,
                metric=metric,
                batch_size=batch_size,
                **kwargs,
            )

        return self._compute_uncached(
            X1,
            X2,
            metric=metric,
            batch_size=batch_size,
            **kwargs,
        )

    def _compute_uncached(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        metric: str,
        batch_size: int | str = "auto",
        **kwargs: Any,
    ) -> NDArray[np.floating]:
        """Compute distances without using the cache.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.
            metric: Distance metric name.
            batch_size: Batch size or mode.
            **kwargs: Metric keyword arguments.

        Returns:
            Distance matrix.
        """
        vectorized = self._vectorized_dispatch.get(metric)
        n1, n2 = len(X1), len(X2)
        dtype = X1.dtype
        n_features = X1.shape[1] if X1.ndim > 1 else 1
        # Estimate the *peak*, including the (n1, n2, d) intermediates the
        # kernel allocates -- not just the output array. Underestimating here
        # is what let the whole-input path run when it should have batched.
        memory_required = self._estimate_memory_usage(n1, n2, dtype, n_features, metric)
        available_memory = min(self.memory_limit_gb, get_available_memory_gb())

        if isinstance(batch_size, str):
            batch_key = batch_size.lower()
        else:
            batch_key = ""

        if batch_key == "stream":
            return self._streaming_computation(X1, X2, metric, **kwargs)

        if batch_size == "auto":
            if memory_required <= available_memory:
                if vectorized is not None:
                    return vectorized(X1, X2, **kwargs)
                batch_size = len(X1)
            else:
                batch_size = self._auto_batch_size(
                    n2,
                    dtype=dtype,
                    n_features=n_features,
                    metric=metric,
                    n_rows=n1,
                )
        elif not isinstance(batch_size, int) or batch_size <= 0:
            raise ValueError(
                "batch_size must be 'auto', 'stream', or a positive integer"
            )

        batch_size = min(int(batch_size), max(1, n1))

        if vectorized is not None and batch_size >= n1:
            return vectorized(X1, X2, **kwargs)

        return self._batched_computation(
            X1,
            X2,
            metric=metric,
            batch_size=int(batch_size),
            vectorized=vectorized,
            **kwargs,
        )

    def _vectorized_euclidean(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Euclidean distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        x1_norm = np.einsum("ij,ij->i", X1, X1)
        x2_norm = np.einsum("ij,ij->i", X2, X2)
        distances = x1_norm[:, None] + x2_norm[None, :] - 2.0 * (X1 @ X2.T)
        np.maximum(distances, 0.0, out=distances)
        result: NDArray[np.floating] = np.sqrt(distances, out=distances)
        return result

    def _vectorized_manhattan(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Manhattan distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        diff = np.abs(X1[:, None, :] - X2[None, :, :])
        result: NDArray[np.floating] = diff.sum(axis=2)
        return result

    def _vectorized_cosine(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized cosine distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        dot = X1 @ X2.T
        norm1 = np.linalg.norm(X1, axis=1)
        norm2 = np.linalg.norm(X2, axis=1)
        denom = norm1[:, None] * norm2[None, :]
        with np.errstate(divide="ignore", invalid="ignore"):
            res = 1.0 - np.where(denom == 0, 0.0, dot / denom)
        return np.nan_to_num(res)

    def _vectorized_hassanat(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Hassanat distance matrix.

        Mirrors :func:`oversampleqa.distance.hassanat_distance`. The
        denominator is ``1 + mx + shift``, which is always ``>= 1``, so no
        division guard is needed.

        Note: this allocates an ``(n1, n2, d)`` intermediate. Memory
        accounting for the batched paths is handled by the caller.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        mn = np.minimum(X1[:, None, :], X2[None, :, :])
        mx = np.maximum(X1[:, None, :], X2[None, :, :])
        shift = np.where(mn < 0.0, -mn, 0.0)
        ratio = (1.0 + mn + shift) / (1.0 + mx + shift)
        result: NDArray[np.floating] = np.sum(1.0 - ratio, axis=-1)
        return result

    def _vectorized_hamming(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Hamming distance matrix.

        Matches the scalar form, which returns the raw **count** of differing
        components rather than SciPy's fraction.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        differing = X1[:, None, :] != X2[None, :, :]
        result: NDArray[np.floating] = differing.sum(axis=-1).astype(float)
        return result

    def _vectorized_jaccard(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Jaccard distance matrix.

        The scalar form casts to ``bool`` and computes set Jaccard, not the
        weighted Ruzicka variant, so this does the same. A pair whose union is
        empty is defined as distance 0.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        # Guarded here as well as in the scalar function: separate code paths.
        if not _is_binary(X1) or not _is_binary(X2):
            raise ValueError(
                "Jaccard distance requires binary inputs: values must be 0 or "
                "1, or a boolean array. Casting other values to bool treats "
                "every non-zero as identical, so distinct points come out at "
                "distance zero. Binarise the features first, choosing the "
                "threshold deliberately."
            )

        b1 = X1.astype(bool)[:, None, :]
        b2 = X2.astype(bool)[None, :, :]
        intersection = np.logical_and(b1, b2).sum(axis=-1)
        union = np.logical_or(b1, b2).sum(axis=-1)
        with np.errstate(divide="ignore", invalid="ignore"):
            similarity = np.where(union == 0, 1.0, intersection / union)
        result: NDArray[np.floating] = 1.0 - similarity
        return result

    @staticmethod
    def _normalise_rows(X: NDArray[np.floating]) -> NDArray[np.floating]:
        """Scale each row to sum to 1, leaving all-zero rows as zeros.

        Mirrors the scalar probability metrics, which divide by the sum unless
        it is zero.
        """
        totals = X.sum(axis=1, keepdims=True)
        with np.errstate(divide="ignore", invalid="ignore"):
            return np.where(totals == 0, 0.0, X / totals)

    def _vectorized_hellinger(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Hellinger distance matrix.

        Rows are normalised once each rather than per pair, which is where the
        saving comes from.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.

        Raises:
            ValueError: If either input contains negative values.
        """
        if np.any(X1 < 0) or np.any(X2 < 0):
            raise ValueError("Hellinger distance requires non-negative inputs")
        root_p = np.sqrt(self._normalise_rows(X1))
        root_q = np.sqrt(self._normalise_rows(X2))
        diff = root_p[:, None, :] - root_q[None, :, :]
        result: NDArray[np.floating] = np.sqrt((diff**2).sum(axis=-1)) / np.sqrt(2.0)
        return result

    def _vectorized_jensen_shannon(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Jensen-Shannon distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.

        Raises:
            ValueError: If either input contains negative values.
        """
        if np.any(X1 < 0) or np.any(X2 < 0):
            raise ValueError("Jensen-Shannon distance requires non-negative inputs")
        p = self._normalise_rows(X1)[:, None, :]
        q = self._normalise_rows(X2)[None, :, :]
        m = 0.5 * (p + q)
        with np.errstate(divide="ignore", invalid="ignore"):
            term_p = np.where(p == 0, 0.0, p * np.log(p / m))
            term_q = np.where(q == 0, 0.0, q * np.log(q / m))
        divergence = 0.5 * (term_p.sum(axis=-1) + term_q.sum(axis=-1))
        result: NDArray[np.floating] = np.sqrt(np.clip(divergence, 0.0, None))
        return result

    def _vectorized_wasserstein(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized 1-D Wasserstein distance matrix.

        Sample-based, like ``energy``: each row is a set of observations. The
        sort each pair needs is hoisted out of the pair loop -- both inputs are
        sorted once, then broadcast -- which is where the win comes from.

        Only valid when both inputs have the same number of columns, which the
        equal-length closed form ``mean|sort(x) - sort(y)|`` requires. The
        caller guarantees this: distance matrices are computed between matrices
        with matching feature counts.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        sorted_1 = np.sort(X1, axis=1)
        sorted_2 = np.sort(X2, axis=1)
        diff = np.abs(sorted_1[:, None, :] - sorted_2[None, :, :])
        result: NDArray[np.floating] = diff.mean(axis=-1)
        return result

    def _vectorized_chebyshev(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Chebyshev distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        diff = np.abs(X1[:, None, :] - X2[None, :, :])
        result: NDArray[np.floating] = diff.max(axis=2)
        return result

    def _vectorized_canberra(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Canberra distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        numerator = np.abs(X1[:, None, :] - X2[None, :, :])
        denominator = np.abs(X1[:, None, :]) + np.abs(X2[None, :, :])
        with np.errstate(divide="ignore", invalid="ignore"):
            ratio = np.where(denominator == 0, 0.0, numerator / denominator)
        result: NDArray[np.floating] = ratio.sum(axis=2)
        return result

    def _vectorized_braycurtis(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Bray-Curtis distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        # Checked here as well as in the scalar implementation: these are two
        # separate code paths, and a guard in one is not a guard in the other.
        if np.any(X1 < 0) or np.any(X2 < 0):
            raise ValueError("Bray-Curtis distance requires non-negative inputs")

        num = np.abs(X1[:, None, :] - X2[None, :, :]).sum(axis=2)
        denom = np.abs(X1[:, None, :] + X2[None, :, :]).sum(axis=2)
        with np.errstate(divide="ignore", invalid="ignore"):
            # denom == 0 means both rows are all-zero, given non-negativity.
            res = np.where(denom == 0, 0.0, num / denom)
        return res

    def _vectorized_correlation(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **_: Any,
    ) -> NDArray[np.floating]:
        """Vectorized correlation distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.

        Returns:
            Distance matrix.
        """
        X1_c = X1 - X1.mean(axis=1, keepdims=True)
        X2_c = X2 - X2.mean(axis=1, keepdims=True)
        dot = X1_c @ X2_c.T
        norm1 = np.linalg.norm(X1_c, axis=1)
        norm2 = np.linalg.norm(X2_c, axis=1)
        denom = norm1[:, None] * norm2[None, :]
        with np.errstate(divide="ignore", invalid="ignore"):
            corr = np.where(denom == 0, 0.0, dot / denom)
        corr = np.nan_to_num(corr)
        result: NDArray[np.floating] = 1.0 - corr
        return result

    def _vectorized_minkowski(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **kwargs: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Minkowski distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.
            **kwargs: Metric keyword arguments (e.g., ``p``).

        Returns:
            Distance matrix.
        """
        p = kwargs.get("p", 3.0)
        diff = np.abs(X1[:, None, :] - X2[None, :, :]) ** p
        result: NDArray[np.floating] = np.sum(diff, axis=2) ** (1.0 / p)
        return result

    def _vectorized_mahalanobis(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        **kwargs: Any,
    ) -> NDArray[np.floating]:
        """Vectorized Mahalanobis distance matrix.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.
            **kwargs: Metric keyword arguments (e.g., ``cov_inv``).

        Returns:
            Distance matrix.
        """
        cov_inv = kwargs.get("cov_inv")
        if cov_inv is None:
            # Matches the scalar path: a silent Euclidean fallback reports one
            # metric under another's name.
            raise ValueError(
                "mahalanobis requires cov_inv: Mahalanobis distance with an "
                "identity covariance is Euclidean distance. Estimate the "
                "inverse from the reference data, e.g. "
                "cov_inv=np.linalg.pinv(np.cov(X, rowvar=False))."
            )
        diff = X1[:, None, :] - X2[None, :, :]
        res = np.einsum("...i,ij,...j->...", diff, cov_inv, diff)
        np.maximum(res, 0.0, out=res)
        result: NDArray[np.floating] = np.sqrt(res, out=res)
        return result

    def _batched_computation(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        metric: str,
        batch_size: int,
        vectorized: Callable[..., NDArray[np.floating]] | None = None,
        **kwargs: Any,
    ) -> NDArray[np.floating]:
        """Compute distances in batches to limit memory usage.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.
            metric: Distance metric name.
            batch_size: Rows per batch.
            vectorized: Optional vectorized kernel.
            **kwargs: Metric keyword arguments.

        Returns:
            Distance matrix.
        """
        result = np.empty((len(X1), len(X2)), dtype=X1.dtype)
        iterator = range(0, len(X1), batch_size)
        iterator = self._progress(iterator, total=len(X1))  # type: ignore[assignment]
        metric_func = self.metric_registry[metric]

        for start in iterator:
            end = min(start + batch_size, len(X1))
            chunk = X1[start:end]
            if vectorized is not None:
                result[start:end] = vectorized(chunk, X2, **kwargs)
            else:
                result[start:end] = self._pairwise(chunk, X2, metric_func, **kwargs)
        return result

    def _streaming_computation(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        metric: str,
        **kwargs: Any,
    ) -> NDArray[np.floating]:
        """Compute distances row-by-row to minimize memory usage.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.
            metric: Distance metric name.
            **kwargs: Metric keyword arguments.

        Returns:
            Distance matrix.
        """
        metric_func = self.metric_registry[metric]
        result = np.empty((len(X1), len(X2)), dtype=X1.dtype)
        iterator = self._progress(range(len(X1)), total=len(X1))
        for idx in iterator:
            row = self._pairwise(X1[idx : idx + 1], X2, metric_func, **kwargs)
            result[idx] = row[0]
        return result

    def _pairwise(
        self,
        X1: NDArray[np.floating],
        X2: NDArray[np.floating],
        metric_func: DistanceCallable,
        **kwargs: Any,
    ) -> NDArray[np.floating]:
        """Compute pairwise distances using a Python loop.

        Args:
            X1: First feature matrix.
            X2: Second feature matrix.
            metric_func: Metric callable.
            **kwargs: Metric keyword arguments.

        Returns:
            Distance matrix.
        """
        dm = np.empty((len(X1), len(X2)), dtype=X1.dtype)
        for i, u in enumerate(X1):
            for j, v in enumerate(X2):
                dm[i, j] = metric_func(u, v, **kwargs)
        return dm

    def _progress(self, iterable: Iterable[int], total: int) -> Iterable[int]:
        """Wrap an iterable with a progress bar if enabled.

        Args:
            iterable: Base iterator.
            total: Total size for progress display.

        Returns:
            Iterator wrapped with tqdm when enabled.
        """
        if not self.show_progress or tqdm is None or total < self.progress_threshold:
            return iterable
        wrapped: Iterable[int] = tqdm(  # pragma: no cover - requires tqdm
            iterable, total=math.ceil(total)
        )
        return wrapped

    def _auto_batch_size(
        self,
        n_cols: int,
        dtype: np.dtype[Any],
        n_features: int = 1,
        metric: str = "",
        n_rows: int = 0,
    ) -> int:
        """Estimate a safe batch size under the memory limit.

        Reserves the accumulating result array before dividing what remains
        into batches, and scales a batch's cost by the metric's intermediate
        multiplier. The previous version allowed every batch to consume the
        entire limit, leaving no headroom for the ``(n1, n2)`` result that lives
        for the whole computation, nor for the ``(batch, n2, d)`` intermediate a
        broadcasting kernel allocates.

        Args:
            n_cols: Number of columns in the distance matrix.
            dtype: Data type of the distance matrix.
            n_features: Feature dimension ``d``.
            metric: Metric name, used to look up the intermediate multiplier.
            n_rows: Total rows, used to reserve the result array.

        Returns:
            Batch size in rows.
        """
        itemsize = np.dtype(dtype).itemsize
        limit_bytes = int(self.memory_limit_gb * (1024**3) * self.safety_factor)

        # The full result array outlives every batch, so subtract it first.
        result_bytes = n_rows * n_cols * itemsize if n_rows else 0
        usable = max(itemsize, limit_bytes - result_bytes)

        # A batch row costs its slice of the output times the kernel's peak
        # multiple, which already includes the output itself.
        row_bytes = max(1, int(n_cols * itemsize * peak_multiple(metric, n_features)))
        return max(1, usable // row_bytes)

    def _estimate_memory_usage(
        self,
        n_rows: int,
        n_cols: int,
        dtype: np.dtype[Any],
        n_features: int = 1,
        metric: str = "",
    ) -> float:
        """Estimate peak memory usage (GB) for a distance computation.

        The output array is ``(n_rows, n_cols)``, but a broadcasting kernel
        also allocates one or more ``(n_rows, n_cols, n_features)``
        intermediates -- so peak use is roughly ``n_features`` times the output,
        multiplied again by how many intermediates the kernel holds at once.
        Ignoring that was how the batching logic got bypassed: the check passed,
        then the kernel allocated far more than the check had permitted.

        Args:
            n_rows: Number of rows.
            n_cols: Number of columns.
            dtype: Data type of the distance matrix.
            n_features: Feature dimension ``d``.
            metric: Metric name; selects the multiplier.

        Returns:
            Estimated peak memory usage in gigabytes.
        """
        itemsize = np.dtype(dtype).itemsize
        result_bytes = n_rows * n_cols * itemsize
        overhead_bytes = (n_rows + n_cols) * itemsize
        peak_bytes = result_bytes * peak_multiple(metric, n_features)
        return (peak_bytes + overhead_bytes) / (1024**3)

    def estimate_memory_gb(
        self,
        n_rows: int,
        n_cols: int,
        dtype: np.dtype[Any] | None = None,
        n_features: int = 1,
        metric: str = "",
    ) -> float:
        """Public helper returning estimated peak footprint of a distance matrix.

        Args:
            n_rows: Number of rows.
            n_cols: Number of columns.
            dtype: Data type of the distance matrix.
            n_features: Feature dimension.
            metric: Metric name; selects the intermediate multiplier.

        Returns:
            Estimated memory usage in gigabytes.
        """
        dtype = dtype or np.dtype(np.float64)
        return self._estimate_memory_usage(n_rows, n_cols, dtype, n_features, metric)

compute_distance_matrix(X1, X2, metric='hassanat', batch_size='auto', **kwargs)

Compute pairwise distances with automatic optimisation.

Parameters

X1, X2: Input matrices of shape (n_samples, n_features). metric: Name of the distance metric registered in metric_registry. batch_size: "auto" selects the largest batch size fitting within memory_limit_gb. An integer enforces a specific chunk length. "stream" yields rows sequentially without storing the full matrix in memory. kwargs: Extra keyword arguments forwarded to the underlying metric.

Source code in src/oversampleqa/optimized_distance.py

def compute_distance_matrix(
    self,
    X1: NDArray[np.floating],
    X2: NDArray[np.floating],
    metric: str = "hassanat",
    batch_size: int | str = "auto",
    **kwargs: Any,
) -> NDArray[np.floating]:
    """Compute pairwise distances with automatic optimisation.

    Parameters
    ----------
    X1, X2:
        Input matrices of shape ``(n_samples, n_features)``.
    metric:
        Name of the distance metric registered in ``metric_registry``.
    batch_size:
        ``"auto"`` selects the largest batch size fitting within
        ``memory_limit_gb``. An integer enforces a specific chunk length.
        ``"stream"`` yields rows sequentially without storing the full
        matrix in memory.
    kwargs:
        Extra keyword arguments forwarded to the underlying metric.
    """
    if metric not in self.metric_registry:
        raise ValueError(f"Unsupported metric '{metric}'")

    X1 = np.asarray(X1, dtype=float)
    X2 = np.asarray(X2, dtype=float)
    n1, n2 = X1.shape[0], X2.shape[0]
    dtype = np.result_type(X1.dtype, X2.dtype, np.float64)
    X1 = X1.astype(dtype, copy=False)
    X2 = X2.astype(dtype, copy=False)

    if n1 == 0 or n2 == 0:
        return np.empty((n1, n2), dtype=dtype)

    if isinstance(batch_size, str):
        batch_key = batch_size.lower()
    else:
        batch_key = ""

    if self.cache is not None and batch_key != "stream":
        return self.cache.cached_distance_matrix(
            optimizer=self,
            X1=X1,
            X2=X2,
            metric=metric,
            batch_size=batch_size,
            **kwargs,
        )

    return self._compute_uncached(
        X1,
        X2,
        metric=metric,
        batch_size=batch_size,
        **kwargs,
    )

estimate_memory_gb(n_rows, n_cols, dtype=None, n_features=1, metric='')

Public helper returning estimated peak footprint of a distance matrix.

Parameters:

Name	Type	Description	Default
n_rows	int	Number of rows.	required
n_cols	int	Number of columns.	required
dtype	dtype[Any] | None	Data type of the distance matrix.	None
n_features	int	Feature dimension.	1
metric	str	Metric name; selects the intermediate multiplier.	''

Returns:

Type	Description
float	Estimated memory usage in gigabytes.

Source code in src/oversampleqa/optimized_distance.py

def estimate_memory_gb(
    self,
    n_rows: int,
    n_cols: int,
    dtype: np.dtype[Any] | None = None,
    n_features: int = 1,
    metric: str = "",
) -> float:
    """Public helper returning estimated peak footprint of a distance matrix.

    Args:
        n_rows: Number of rows.
        n_cols: Number of columns.
        dtype: Data type of the distance matrix.
        n_features: Feature dimension.
        metric: Metric name; selects the intermediate multiplier.

    Returns:
        Estimated memory usage in gigabytes.
    """
    dtype = dtype or np.dtype(np.float64)
    return self._estimate_memory_usage(n_rows, n_cols, dtype, n_features, metric)

AxiomReport dataclass

Which metric axioms a callable satisfied, and how it failed.

Source code in src/oversampleqa/plugin_contract.py

@dataclass(frozen=True)
class AxiomReport:
    """Which metric axioms a callable satisfied, and how it failed."""

    identity: bool
    identity_of_indiscernibles: bool
    symmetry: bool
    non_negativity: bool
    finiteness: bool
    failures: tuple[str, ...] = ()

    @property
    def ok(self) -> bool:
        """Whether every checked axiom held."""
        return not self.failures

    def __bool__(self) -> bool:
        """Truthy when every axiom held."""
        return self.ok

ok property

Whether every checked axiom held.

__bool__()

Truthy when every axiom held.

Source code in src/oversampleqa/plugin_contract.py

def __bool__(self) -> bool:
    """Truthy when every axiom held."""
    return self.ok

MetricPlugin

Bases: Protocol

A distance metric: two vectors in, one float out.

Source code in src/oversampleqa/plugin_contract.py

@runtime_checkable
class MetricPlugin(Protocol):
    """A distance metric: two vectors in, one float out."""

    def __call__(
        self, x1: NDArray[np.floating], x2: NDArray[np.floating], **kwargs: Any
    ) -> float:
        """Return the distance between ``x1`` and ``x2``."""
        ...

__call__(x1, x2, **kwargs)

Return the distance between x1 and x2.

Source code in src/oversampleqa/plugin_contract.py

def __call__(
    self, x1: NDArray[np.floating], x2: NDArray[np.floating], **kwargs: Any
) -> float:
    """Return the distance between ``x1`` and ``x2``."""
    ...

RunMetadata dataclass

Everything needed to reproduce and audit a run.

A number without its provenance is not a result. This records the package and dependency versions, the sampler and its parameters, the seed, and a hash of the data -- so a report exported today can be checked against a rerun in a year, and a mismatch localised to whichever of those changed.

Source code in src/oversampleqa/reports.py

@dataclass(frozen=True)
class RunMetadata:
    """Everything needed to reproduce and audit a run.

    A number without its provenance is not a result. This records the package
    and dependency versions, the sampler and its parameters, the seed, and a
    hash of the data -- so a report exported today can be checked against a
    rerun in a year, and a mismatch localised to whichever of those changed.
    """

    oversampler: str = ""
    oversampler_params: dict[str, Any] = field(default_factory=dict)
    metric: str = "hassanat"
    hidden_ratio: float = 0.1
    reference: str = "hidden_minority"
    random_state: int | None = None
    n_repeats: int = 1
    dataset: str = ""
    dataset_hash: str = ""
    n_samples: int = 0
    n_features: int = 0
    minority_label: int | None = None
    oversampleqa_version: str = ""
    numpy_version: str = ""
    sklearn_version: str = ""
    imblearn_version: str = ""
    timestamp: str = ""

    @classmethod
    def capture(
        cls,
        X: NDArray[np.floating],
        y: NDArray[np.integer],
        oversampler: Any,
        *,
        minority_label: int | None = None,
        metric: str = "hassanat",
        hidden_ratio: float = 0.1,
        reference: str = "hidden_minority",
        random_state: int | None = None,
        n_repeats: int = 1,
    ) -> RunMetadata:
        """Collect metadata for a run about to happen, or just completed."""
        import sklearn
        from imblearn import __version__ as imblearn_version

        from . import __version__ as package_version

        params: dict[str, Any] = {}
        if hasattr(oversampler, "get_params"):
            params = {k: repr(v) for k, v in oversampler.get_params().items()}

        X_arr = np.asarray(X)
        return cls(
            oversampler=type(oversampler).__name__,
            oversampler_params=params,
            metric=metric,
            hidden_ratio=hidden_ratio,
            reference=reference,
            random_state=random_state,
            n_repeats=n_repeats,
            dataset_hash=_dataset_hash(X, y),
            n_samples=int(X_arr.shape[0]),
            n_features=int(X_arr.shape[1]) if X_arr.ndim > 1 else 1,
            minority_label=minority_label,
            oversampleqa_version=package_version,
            numpy_version=np.__version__,
            sklearn_version=sklearn.__version__,
            imblearn_version=imblearn_version,
            timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
        )

    def to_dict(self) -> dict[str, Any]:
        """JSON-safe mapping."""
        payload: dict[str, Any] = _json_safe(
            {
                "oversampler": self.oversampler,
                "oversampler_params": self.oversampler_params,
                "metric": self.metric,
                "hidden_ratio": self.hidden_ratio,
                "reference": self.reference,
                "random_state": self.random_state,
                "n_repeats": self.n_repeats,
                "dataset_hash": self.dataset_hash,
                "n_samples": self.n_samples,
                "n_features": self.n_features,
                "minority_label": self.minority_label,
                "oversampleqa_version": self.oversampleqa_version,
                "numpy_version": self.numpy_version,
                "sklearn_version": self.sklearn_version,
                "imblearn_version": self.imblearn_version,
                "timestamp": self.timestamp,
            }
        )
        return payload

    @classmethod
    def from_dict(cls, payload: dict[str, Any]) -> RunMetadata:
        """Rebuild from :meth:`to_dict` output, ignoring unknown keys."""
        known = set(cls.__dataclass_fields__)
        return cls(**{k: v for k, v in payload.items() if k in known})

capture(X, y, oversampler, *, minority_label=None, metric='hassanat', hidden_ratio=0.1, reference='hidden_minority', random_state=None, n_repeats=1) classmethod

Collect metadata for a run about to happen, or just completed.

Source code in src/oversampleqa/reports.py

@classmethod
def capture(
    cls,
    X: NDArray[np.floating],
    y: NDArray[np.integer],
    oversampler: Any,
    *,
    minority_label: int | None = None,
    metric: str = "hassanat",
    hidden_ratio: float = 0.1,
    reference: str = "hidden_minority",
    random_state: int | None = None,
    n_repeats: int = 1,
) -> RunMetadata:
    """Collect metadata for a run about to happen, or just completed."""
    import sklearn
    from imblearn import __version__ as imblearn_version

    from . import __version__ as package_version

    params: dict[str, Any] = {}
    if hasattr(oversampler, "get_params"):
        params = {k: repr(v) for k, v in oversampler.get_params().items()}

    X_arr = np.asarray(X)
    return cls(
        oversampler=type(oversampler).__name__,
        oversampler_params=params,
        metric=metric,
        hidden_ratio=hidden_ratio,
        reference=reference,
        random_state=random_state,
        n_repeats=n_repeats,
        dataset_hash=_dataset_hash(X, y),
        n_samples=int(X_arr.shape[0]),
        n_features=int(X_arr.shape[1]) if X_arr.ndim > 1 else 1,
        minority_label=minority_label,
        oversampleqa_version=package_version,
        numpy_version=np.__version__,
        sklearn_version=sklearn.__version__,
        imblearn_version=imblearn_version,
        timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
    )

to_dict()

JSON-safe mapping.

Source code in src/oversampleqa/reports.py

def to_dict(self) -> dict[str, Any]:
    """JSON-safe mapping."""
    payload: dict[str, Any] = _json_safe(
        {
            "oversampler": self.oversampler,
            "oversampler_params": self.oversampler_params,
            "metric": self.metric,
            "hidden_ratio": self.hidden_ratio,
            "reference": self.reference,
            "random_state": self.random_state,
            "n_repeats": self.n_repeats,
            "dataset_hash": self.dataset_hash,
            "n_samples": self.n_samples,
            "n_features": self.n_features,
            "minority_label": self.minority_label,
            "oversampleqa_version": self.oversampleqa_version,
            "numpy_version": self.numpy_version,
            "sklearn_version": self.sklearn_version,
            "imblearn_version": self.imblearn_version,
            "timestamp": self.timestamp,
        }
    )
    return payload

from_dict(payload) classmethod

Rebuild from :meth:to_dict output, ignoring unknown keys.

Source code in src/oversampleqa/reports.py

@classmethod
def from_dict(cls, payload: dict[str, Any]) -> RunMetadata:
    """Rebuild from :meth:`to_dict` output, ignoring unknown keys."""
    known = set(cls.__dataclass_fields__)
    return cls(**{k: v for k, v in payload.items() if k in known})

ValidationReport dataclass

Everything known about one oversampler on one dataset.

calibration, inference and fidelity are optional because each costs real time: the calibration fits nothing but resamples repeatedly, the two-sample tests permute, and the fidelity suite can fit models. A report with only error_rate and details is the cheap default.

Source code in src/oversampleqa/reports.py

@dataclass(frozen=True)
class ValidationReport:
    """Everything known about one oversampler on one dataset.

    ``calibration``, ``inference`` and ``fidelity`` are optional because each
    costs real time: the calibration fits nothing but resamples repeatedly, the
    two-sample tests permute, and the fidelity suite can fit models. A report
    with only ``error_rate`` and ``details`` is the cheap default.
    """

    error_rate: float
    metadata: RunMetadata
    details: Any = None
    calibration: Any = None
    inference: Any = None
    fidelity: Any = None
    schema_version: str = SCHEMA_VERSION

    def to_dict(self) -> dict[str, Any]:
        """JSON-serialisable mapping of the whole report.

        Non-finite floats become ``null``; see :func:`_json_safe`.
        """
        payload: dict[str, Any] = {
            "schema_version": self.schema_version,
            "error_rate": _json_safe(self.error_rate),
            "metadata": self.metadata.to_dict(),
        }
        for name in ("details", "calibration", "inference", "fidelity"):
            component = getattr(self, name)
            if component is None:
                payload[name] = None
            elif hasattr(component, "to_dict"):
                payload[name] = _json_safe(component.to_dict())
            else:  # pragma: no cover - defensive
                payload[name] = _json_safe(component)
        return payload

    @classmethod
    def from_dict(cls, payload: dict[str, Any]) -> ValidationReport:
        """Rebuild from :meth:`to_dict` output.

        Components come back as plain dicts rather than their original
        dataclasses: the export is the interchange format, and rehydrating each
        component type would couple this module to every one of them. Round
        trips are therefore compared on ``to_dict()``, which is what a consumer
        actually reads.
        """
        version = payload.get("schema_version", "0")
        if version.split(".")[0] != SCHEMA_VERSION.split(".")[0]:
            raise ValueError(
                f"report schema version {version} is not compatible with "
                f"{SCHEMA_VERSION}; a major-version change means a field was "
                "removed or changed meaning, so this cannot be read safely"
            )
        return cls(
            error_rate=(
                float("nan")
                if payload.get("error_rate") is None
                else float(payload["error_rate"])
            ),
            metadata=RunMetadata.from_dict(payload.get("metadata", {})),
            details=payload.get("details"),
            calibration=payload.get("calibration"),
            inference=payload.get("inference"),
            fidelity=payload.get("fidelity"),
            schema_version=version,
        )

    def to_json(self, indent: int = 2) -> str:
        """Serialise to JSON. ``allow_nan=False`` guarantees valid output."""
        return strict_json_dumps(self.to_dict(), indent=indent)

    def to_frame(self) -> pd.DataFrame:
        """Tidy one-row frame with every scalar flattened."""
        flat: dict[str, Any] = {
            "schema_version": self.schema_version,
            "error_rate": self.error_rate,
            # `dataset` is whatever the caller named it and is empty when they
            # named nothing: this surface validates arrays, not a file, so it
            # has no name of its own. `dataset_hash` is the identity that is
            # always present, and is promoted out of the `meta_` block because
            # a row nobody can trace back to its data is not much of a record.
            "dataset": self.metadata.dataset,
            "dataset_hash": self.metadata.dataset_hash,
            "oversampler": self.metadata.oversampler,
            "metric": self.metadata.metric,
            "hidden_ratio": self.metadata.hidden_ratio,
            "reference": self.metadata.reference,
            "random_state": self.metadata.random_state,
            "n_repeats": self.metadata.n_repeats,
            "minority_label": self.metadata.minority_label,
            "oversampleqa_version": self.metadata.oversampleqa_version,
        }
        flat.update(
            {
                f"meta_{k}": v
                for k, v in self.metadata.to_dict().items()
                if not isinstance(v, (dict, list))
            }
        )
        for name in ("details", "calibration", "inference", "fidelity"):
            component = getattr(self, name)
            if component is None or not hasattr(component, "to_dict"):
                continue
            for key, value in component.to_dict().items():
                if not isinstance(value, (dict, list, tuple, np.ndarray)):
                    flat[f"{name}_{key}"] = value
        return pd.DataFrame([flat])

    def __rich__(self) -> str:
        """Compact CLI rendering."""
        lines = [
            f"[bold]OversampleQA report[/bold] (schema {self.schema_version})",
            f"  error rate     {self.error_rate:.4f}",
            f"  oversampler    {self.metadata.oversampler}",
            f"  metric         {self.metadata.metric}",
            f"  random_state   {self.metadata.random_state}",
            f"  dataset        {self.metadata.dataset_hash} "
            f"({self.metadata.n_samples}x{self.metadata.n_features})",
        ]
        if self.calibration is not None and hasattr(self.calibration, "interpret"):
            lines.append(f"  calibration    {self.calibration.interpret()}")
        if self.fidelity is not None and hasattr(self.fidelity, "interpret"):
            lines.extend(f"  fidelity       {n}" for n in self.fidelity.interpret())
        return "\n".join(lines)

    def with_components(self, **components: Any) -> ValidationReport:
        """Return a copy carrying additional components."""
        return replace(self, **components)

to_dict()

JSON-serialisable mapping of the whole report.

Non-finite floats become null; see :func:_json_safe.

Source code in src/oversampleqa/reports.py

def to_dict(self) -> dict[str, Any]:
    """JSON-serialisable mapping of the whole report.

    Non-finite floats become ``null``; see :func:`_json_safe`.
    """
    payload: dict[str, Any] = {
        "schema_version": self.schema_version,
        "error_rate": _json_safe(self.error_rate),
        "metadata": self.metadata.to_dict(),
    }
    for name in ("details", "calibration", "inference", "fidelity"):
        component = getattr(self, name)
        if component is None:
            payload[name] = None
        elif hasattr(component, "to_dict"):
            payload[name] = _json_safe(component.to_dict())
        else:  # pragma: no cover - defensive
            payload[name] = _json_safe(component)
    return payload

from_dict(payload) classmethod

Rebuild from :meth:to_dict output.

Components come back as plain dicts rather than their original dataclasses: the export is the interchange format, and rehydrating each component type would couple this module to every one of them. Round trips are therefore compared on to_dict(), which is what a consumer actually reads.

Source code in src/oversampleqa/reports.py

@classmethod
def from_dict(cls, payload: dict[str, Any]) -> ValidationReport:
    """Rebuild from :meth:`to_dict` output.

    Components come back as plain dicts rather than their original
    dataclasses: the export is the interchange format, and rehydrating each
    component type would couple this module to every one of them. Round
    trips are therefore compared on ``to_dict()``, which is what a consumer
    actually reads.
    """
    version = payload.get("schema_version", "0")
    if version.split(".")[0] != SCHEMA_VERSION.split(".")[0]:
        raise ValueError(
            f"report schema version {version} is not compatible with "
            f"{SCHEMA_VERSION}; a major-version change means a field was "
            "removed or changed meaning, so this cannot be read safely"
        )
    return cls(
        error_rate=(
            float("nan")
            if payload.get("error_rate") is None
            else float(payload["error_rate"])
        ),
        metadata=RunMetadata.from_dict(payload.get("metadata", {})),
        details=payload.get("details"),
        calibration=payload.get("calibration"),
        inference=payload.get("inference"),
        fidelity=payload.get("fidelity"),
        schema_version=version,
    )

to_json(indent=2)

Serialise to JSON. allow_nan=False guarantees valid output.

Source code in src/oversampleqa/reports.py

def to_json(self, indent: int = 2) -> str:
    """Serialise to JSON. ``allow_nan=False`` guarantees valid output."""
    return strict_json_dumps(self.to_dict(), indent=indent)

to_frame()

Tidy one-row frame with every scalar flattened.

Source code in src/oversampleqa/reports.py

def to_frame(self) -> pd.DataFrame:
    """Tidy one-row frame with every scalar flattened."""
    flat: dict[str, Any] = {
        "schema_version": self.schema_version,
        "error_rate": self.error_rate,
        # `dataset` is whatever the caller named it and is empty when they
        # named nothing: this surface validates arrays, not a file, so it
        # has no name of its own. `dataset_hash` is the identity that is
        # always present, and is promoted out of the `meta_` block because
        # a row nobody can trace back to its data is not much of a record.
        "dataset": self.metadata.dataset,
        "dataset_hash": self.metadata.dataset_hash,
        "oversampler": self.metadata.oversampler,
        "metric": self.metadata.metric,
        "hidden_ratio": self.metadata.hidden_ratio,
        "reference": self.metadata.reference,
        "random_state": self.metadata.random_state,
        "n_repeats": self.metadata.n_repeats,
        "minority_label": self.metadata.minority_label,
        "oversampleqa_version": self.metadata.oversampleqa_version,
    }
    flat.update(
        {
            f"meta_{k}": v
            for k, v in self.metadata.to_dict().items()
            if not isinstance(v, (dict, list))
        }
    )
    for name in ("details", "calibration", "inference", "fidelity"):
        component = getattr(self, name)
        if component is None or not hasattr(component, "to_dict"):
            continue
        for key, value in component.to_dict().items():
            if not isinstance(value, (dict, list, tuple, np.ndarray)):
                flat[f"{name}_{key}"] = value
    return pd.DataFrame([flat])

__rich__()

Compact CLI rendering.

Source code in src/oversampleqa/reports.py

def __rich__(self) -> str:
    """Compact CLI rendering."""
    lines = [
        f"[bold]OversampleQA report[/bold] (schema {self.schema_version})",
        f"  error rate     {self.error_rate:.4f}",
        f"  oversampler    {self.metadata.oversampler}",
        f"  metric         {self.metadata.metric}",
        f"  random_state   {self.metadata.random_state}",
        f"  dataset        {self.metadata.dataset_hash} "
        f"({self.metadata.n_samples}x{self.metadata.n_features})",
    ]
    if self.calibration is not None and hasattr(self.calibration, "interpret"):
        lines.append(f"  calibration    {self.calibration.interpret()}")
    if self.fidelity is not None and hasattr(self.fidelity, "interpret"):
        lines.extend(f"  fidelity       {n}" for n in self.fidelity.interpret())
    return "\n".join(lines)

with_components(**components)

Return a copy carrying additional components.

Source code in src/oversampleqa/reports.py

def with_components(self, **components: Any) -> ValidationReport:
    """Return a copy carrying additional components."""
    return replace(self, **components)

PydanticValidationConfig

Bases: BaseModel

Runtime validation for configuration parameters.

Source code in src/oversampleqa/typed_validator.py

class PydanticValidationConfig(BaseModel):
    """Runtime validation for configuration parameters."""

    hidden_ratio: float = Field(default=0.1, gt=0.0, lt=1.0)
    metric: str = Field(default="hassanat")
    return_details: bool = Field(default=False)
    random_state: int | None = Field(default=None)

    @field_validator("metric")
    def validate_metric(cls, value: str) -> str:
        """Validate that the metric is supported.

        Accepts registered plugin metrics as well as built-ins. Checking the
        built-in table alone rejected a plugin metric at config construction --
        before any validation ran -- even though ``distance_matrix`` would
        compute it.

        Args:
            value: Metric name.

        Returns:
            The validated metric name.

        Raises:
            ValueError: If the metric is neither built in nor registered.
        """
        resolve_metric(value)
        return value

    @field_validator("random_state")
    def validate_random_state(cls, value: int | None) -> int | None:
        """Validate random_state bounds when provided.

        Args:
            value: Optional random state.

        Returns:
            The validated random state.
        """
        if value is not None and not (0 <= value < 2**31):
            raise ValueError("random_state must be between 0 and 2**31 - 1")
        return value

validate_metric(value)

Validate that the metric is supported.

Accepts registered plugin metrics as well as built-ins. Checking the built-in table alone rejected a plugin metric at config construction -- before any validation ran -- even though distance_matrix would compute it.

Parameters:

Name	Type	Description	Default
value	str	Metric name.	required

Returns:

Type	Description
str	The validated metric name.

Raises:

Type	Description
ValueError	If the metric is neither built in nor registered.

Source code in src/oversampleqa/typed_validator.py

@field_validator("metric")
def validate_metric(cls, value: str) -> str:
    """Validate that the metric is supported.

    Accepts registered plugin metrics as well as built-ins. Checking the
    built-in table alone rejected a plugin metric at config construction --
    before any validation ran -- even though ``distance_matrix`` would
    compute it.

    Args:
        value: Metric name.

    Returns:
        The validated metric name.

    Raises:
        ValueError: If the metric is neither built in nor registered.
    """
    resolve_metric(value)
    return value

validate_random_state(value)

Validate random_state bounds when provided.

Parameters:

Name	Type	Description	Default
value	int | None	Optional random state.	required

Returns:

Type	Description
int | None	The validated random state.

Source code in src/oversampleqa/typed_validator.py

@field_validator("random_state")
def validate_random_state(cls, value: int | None) -> int | None:
    """Validate random_state bounds when provided.

    Args:
        value: Optional random state.

    Returns:
        The validated random state.
    """
    if value is not None and not (0 <= value < 2**31):
        raise ValueError("random_state must be between 0 and 2**31 - 1")
    return value

TypedValidator

Bases: BaseValidator[ValidationResult]

Type-safe validator wrapper with runtime validation.

Source code in src/oversampleqa/typed_validator.py

class TypedValidator(BaseValidator[ValidationResult]):
    """Type-safe validator wrapper with runtime validation."""

    def __init__(self, mode: ValidationMode = ValidationMode.STANDARD) -> None:
        self.mode = mode

    @overload
    def validate(
        self,
        X: FloatArray,
        y: IntArray,
        minority_label: int,
        oversampler: OversamplerProtocol,
        config: ValidationConfig,
    ) -> ValidationResult:
        """Validate using a prebuilt ValidationConfig.

        Args:
            X: Feature matrix.
            y: Target labels.
            minority_label: Minority class label.
            oversampler: Oversampler instance.
            config: ValidationConfig.

        Returns:
            ValidationResult.
        """

    @overload
    def validate(
        self,
        X: FloatArray,
        y: IntArray,
        minority_label: int,
        oversampler: OversamplerProtocol,
        *,
        hidden_ratio: float = 0.1,
        metric: str = "hassanat",
        return_details: bool = False,
        random_state: int | None = None,
    ) -> ValidationResult:
        """Validate using keyword configuration parameters.

        Args:
            X: Feature matrix.
            y: Target labels.
            minority_label: Minority class label.
            oversampler: Oversampler instance.
            hidden_ratio: Fraction of majority to hide.
            metric: Distance metric name.
            return_details: Whether to include distance matrices.
            random_state: Optional random seed.

        Returns:
            ValidationResult.
        """

    def validate(
        self,
        X: FloatArray,
        y: IntArray,
        minority_label: int,
        oversampler: OversamplerProtocol,
        config: ValidationConfig | None = None,
        **kwargs: Any,
    ) -> ValidationResult:
        """Validate oversampling with typed configuration.

        Args:
            X: Feature matrix.
            y: Target labels.
            minority_label: Minority class label.
            oversampler: Oversampler instance.
            config: ValidationConfig, or None to build from kwargs.
            **kwargs: ValidationConfig fields when ``config`` is None.

        Returns:
            ValidationResult with error rate and optional details.
        """
        if config is None:
            parsed = PydanticValidationConfig(**kwargs)
            metric_name = cast(MetricName, parsed.metric)
            config = ValidationConfig(
                hidden_ratio=parsed.hidden_ratio,
                metric=metric_name,
                return_details=parsed.return_details,
                random_state=parsed.random_state,
            )

        self._validate_inputs(X, y, minority_label, oversampler, config)

        if self.mode == ValidationMode.MEMORY_EFFICIENT:
            return self._validate_standard(X, y, minority_label, oversampler, config)
        if self.mode == ValidationMode.PARALLEL:
            # Future: add parallel implementation
            return self._validate_standard(X, y, minority_label, oversampler, config)
        if self.mode == ValidationMode.ASYNC:
            # Deliberately raises rather than driving a loop.
            #
            # This used to call get_event_loop().run_until_complete(), which
            # throws if a loop is already running -- so it broke in Jupyter and
            # in any async host, exactly where someone would reach for it.
            #
            # Driving it correctly would not help either: the work is CPU-bound
            # NumPy, so asyncio buys no concurrency at all. It only moves the
            # call onto an executor thread and waits for it. Callers who want
            # a coroutine should await validate_async directly; callers who want
            # parallelism want processes, not an event loop.
            raise ConfigurationError(
                "ValidationMode.ASYNC cannot be driven from a synchronous call. "
                "Validation is CPU-bound NumPy, so asyncio provides no "
                "concurrency for it. Await `validate_async(...)` directly from "
                "async code, or use ValidationMode.STANDARD here and parallelise "
                "across repeats or datasets instead."
            )
        return self._validate_standard(X, y, minority_label, oversampler, config)

    async def validate_async(
        self,
        X: FloatArray,
        y: IntArray,
        minority_label: int,
        oversampler: OversamplerProtocol,
        config: ValidationConfig,
    ) -> ValidationResult:
        """Async wrapper around validate using an executor.

        Args:
            X: Feature matrix.
            y: Target labels.
            minority_label: Minority class label.
            oversampler: Oversampler instance.
            config: ValidationConfig.

        Returns:
            ValidationResult.
        """
        loop = asyncio.get_running_loop()
        return await loop.run_in_executor(
            None, self.validate, X, y, minority_label, oversampler, config
        )

    def _validate_inputs(
        self,
        X: FloatArray,
        y: IntArray,
        minority_label: int,
        oversampler: OversamplerProtocol,
        config: ValidationConfig,
    ) -> None:
        """Validate input arrays and configuration.

        Args:
            X: Feature matrix.
            y: Target labels.
            minority_label: Minority class label.
            oversampler: Oversampler instance.
            config: ValidationConfig.
        """
        if not isinstance(X, np.ndarray) or not np.issubdtype(X.dtype, np.floating):
            raise ValidationError("X must be a floating-point numpy array")
        if not isinstance(y, np.ndarray) or not np.issubdtype(y.dtype, np.integer):
            raise ValidationError("y must be an integer numpy array")
        if X.shape[0] != y.shape[0]:
            raise ValidationError("X and y must have the same number of rows")
        if minority_label not in y:
            raise ValidationError(f"minority_label {minority_label} not present in y")
        if not hasattr(oversampler, "fit_resample"):
            raise ValidationError("oversampler must implement fit_resample")
        # Built-ins *and* registered plugins. Checking _METRICS alone rejected a
        # plugin metric that distance_matrix would happily have computed.
        try:
            resolve_metric(config.metric)
        except ValueError as exc:
            raise MetricError(str(exc)) from None

    def _validate_standard(
        self,
        X: FloatArray,
        y: IntArray,
        minority_label: int,
        oversampler: OversamplerProtocol,
        config: ValidationConfig,
    ) -> ValidationResult:
        """Execute the standard validation pipeline.

        Args:
            X: Feature matrix.
            y: Target labels.
            minority_label: Minority class label.
            oversampler: Oversampler instance.
            config: ValidationConfig.

        Returns:
            ValidationResult.
        """
        from .validator import validate_oversampling

        try:
            if config.return_details:
                details = validate_oversampling(
                    X,
                    y,
                    minority_label,
                    oversampler,
                    hidden_ratio=config.hidden_ratio,
                    metric=config.metric,
                    return_details=True,
                    reference=config.reference,
                    random_state=config.random_state,
                    n_repeats=config.n_repeats,
                )
                # return_details=True always yields ValidationDetails; the
                # runtime check narrows the union without suppressing the type.
                if not isinstance(details, ValidationDetails):
                    raise TypeError(
                        "validate_oversampling(return_details=True) must return "
                        f"ValidationDetails, got {type(details).__name__}"
                    )
                n_synthetic = details.n_synthetic
                ci = self._wilson_confidence_interval(
                    details.error_rate, max(n_synthetic, 1)
                )
                return ValidationResult(
                    error_rate=details.error_rate,
                    n_errors=details.n_errors,
                    n_synthetic=n_synthetic,
                    confidence_interval=ci,
                    metadata={
                        "distance_matrices": {
                            "hidden": details.dist_hidden,
                            "minority": details.dist_min,
                        },
                        "n_ties": details.n_ties,
                        "duplication_rate": details.duplication_rate,
                        "reference": details.reference,
                        "random_state": config.random_state,
                        "n_repeats": details.n_repeats,
                        "rates": details.rates,
                        "std": details.std,
                        "repeat_interval": details.interval,
                    },
                )
            error_rate = validate_oversampling(
                X,
                y,
                minority_label,
                oversampler,
                hidden_ratio=config.hidden_ratio,
                metric=config.metric,
                return_details=False,
                reference=config.reference,
                random_state=config.random_state,
                n_repeats=config.n_repeats,
            )
            if isinstance(error_rate, ValidationDetails):  # pragma: no cover
                raise ValidationError(
                    "validate_oversampling(return_details=False) must return a float"
                )
            ci = self._wilson_confidence_interval(error_rate, len(y))
            return ValidationResult(
                error_rate=error_rate,
                n_errors=0,
                n_synthetic=0,
                confidence_interval=ci,
                metadata={},
            )
        except Exception as exc:  # pragma: no cover - defensive
            raise ValidationError(f"Validation failed: {exc}") from exc

    @staticmethod
    def _wilson_confidence_interval(
        rate: float, n: int, z: float = 1.96
    ) -> tuple[float, float]:
        """Compute a Wilson score interval for a binomial proportion.

        Replaces the previous Wald interval, which is unreliable exactly where
        this package spends most of its time. Wald is symmetric around the
        estimate, so near ``rate = 0`` it produces a degenerate zero-width
        interval (its standard error vanishes) and can extend below zero. Error
        rates near zero are the common case here. Wilson stays inside ``[0, 1]``
        and keeps sensible width at the boundaries.

        .. warning::

           This assumes **independent Bernoulli trials**. Synthetic points
           interpolated from shared parent points are not independent, so a real
           interval is wider than this one. Treat it as a lower bound on
           uncertainty. Proper inference is not yet implemented; see
           :func:`~oversampleqa.validate_oversampling`'s ``n_repeats``, which
           measures the variability of the hold-out split instead.

        Args:
            rate: Estimated proportion.
            n: Sample size.
            z: Z-score for the confidence level.

        Returns:
            Lower and upper confidence bounds, both within ``[0, 1]``.
        """
        if n <= 0:
            return (0.0, 1.0)
        denominator = 1.0 + z**2 / n
        centre = (rate + z**2 / (2 * n)) / denominator
        margin = z * math.sqrt(rate * (1 - rate) / n + z**2 / (4 * n**2)) / denominator
        return (max(0.0, centre - margin), min(1.0, centre + margin))

validate(X, y, minority_label, oversampler, config=None, **kwargs)

validate(
    X: FloatArray,
    y: IntArray,
    minority_label: int,
    oversampler: OversamplerProtocol,
    config: ValidationConfig,
) -> ValidationResult

validate(
    X: FloatArray,
    y: IntArray,
    minority_label: int,
    oversampler: OversamplerProtocol,
    *,
    hidden_ratio: float = 0.1,
    metric: str = "hassanat",
    return_details: bool = False,
    random_state: int | None = None,
) -> ValidationResult

Validate oversampling with typed configuration.

Parameters:

Name	Type	Description	Default
X	FloatArray	Feature matrix.	required
y	IntArray	Target labels.	required
minority_label	int	Minority class label.	required
oversampler	OversamplerProtocol	Oversampler instance.	required
config	ValidationConfig | None	ValidationConfig, or None to build from kwargs.	None
**kwargs	Any	ValidationConfig fields when config is None.	{}

Returns:

Type	Description
ValidationResult	ValidationResult with error rate and optional details.

Source code in src/oversampleqa/typed_validator.py

def validate(
    self,
    X: FloatArray,
    y: IntArray,
    minority_label: int,
    oversampler: OversamplerProtocol,
    config: ValidationConfig | None = None,
    **kwargs: Any,
) -> ValidationResult:
    """Validate oversampling with typed configuration.

    Args:
        X: Feature matrix.
        y: Target labels.
        minority_label: Minority class label.
        oversampler: Oversampler instance.
        config: ValidationConfig, or None to build from kwargs.
        **kwargs: ValidationConfig fields when ``config`` is None.

    Returns:
        ValidationResult with error rate and optional details.
    """
    if config is None:
        parsed = PydanticValidationConfig(**kwargs)
        metric_name = cast(MetricName, parsed.metric)
        config = ValidationConfig(
            hidden_ratio=parsed.hidden_ratio,
            metric=metric_name,
            return_details=parsed.return_details,
            random_state=parsed.random_state,
        )

    self._validate_inputs(X, y, minority_label, oversampler, config)

    if self.mode == ValidationMode.MEMORY_EFFICIENT:
        return self._validate_standard(X, y, minority_label, oversampler, config)
    if self.mode == ValidationMode.PARALLEL:
        # Future: add parallel implementation
        return self._validate_standard(X, y, minority_label, oversampler, config)
    if self.mode == ValidationMode.ASYNC:
        # Deliberately raises rather than driving a loop.
        #
        # This used to call get_event_loop().run_until_complete(), which
        # throws if a loop is already running -- so it broke in Jupyter and
        # in any async host, exactly where someone would reach for it.
        #
        # Driving it correctly would not help either: the work is CPU-bound
        # NumPy, so asyncio buys no concurrency at all. It only moves the
        # call onto an executor thread and waits for it. Callers who want
        # a coroutine should await validate_async directly; callers who want
        # parallelism want processes, not an event loop.
        raise ConfigurationError(
            "ValidationMode.ASYNC cannot be driven from a synchronous call. "
            "Validation is CPU-bound NumPy, so asyncio provides no "
            "concurrency for it. Await `validate_async(...)` directly from "
            "async code, or use ValidationMode.STANDARD here and parallelise "
            "across repeats or datasets instead."
        )
    return self._validate_standard(X, y, minority_label, oversampler, config)

validate_async(X, y, minority_label, oversampler, config) async

Async wrapper around validate using an executor.

Parameters:

Name	Type	Description	Default
X	FloatArray	Feature matrix.	required
y	IntArray	Target labels.	required
minority_label	int	Minority class label.	required
oversampler	OversamplerProtocol	Oversampler instance.	required
config	ValidationConfig	ValidationConfig.	required

Returns:

Type	Description
ValidationResult	ValidationResult.

Source code in src/oversampleqa/typed_validator.py

async def validate_async(
    self,
    X: FloatArray,
    y: IntArray,
    minority_label: int,
    oversampler: OversamplerProtocol,
    config: ValidationConfig,
) -> ValidationResult:
    """Async wrapper around validate using an executor.

    Args:
        X: Feature matrix.
        y: Target labels.
        minority_label: Minority class label.
        oversampler: Oversampler instance.
        config: ValidationConfig.

    Returns:
        ValidationResult.
    """
    loop = asyncio.get_running_loop()
    return await loop.run_in_executor(
        None, self.validate, X, y, minority_label, oversampler, config
    )

BenchmarkConfig dataclass

Configuration for benchmarking experiments.

Source code in src/oversampleqa/types.py

@dataclass(frozen=True)
class BenchmarkConfig:
    """Configuration for benchmarking experiments."""

    n_runs: int = 10
    hidden_ratios: list[float] = field(default_factory=lambda: [0.1, 0.25, 0.5])
    metrics: list[str] = field(default_factory=lambda: ["hassanat", "euclidean"])
    validation_mode: ValidationMode = ValidationMode.STANDARD
    n_jobs: int = 1

    def __post_init__(self) -> None:
        if self.n_runs <= 0:
            raise ValueError("n_runs must be positive")
        if any(ratio <= 0 or ratio >= 1 for ratio in self.hidden_ratios):
            raise ValueError("All hidden ratios must be in (0, 1)")

ConfigurationError

Bases: OversampleQAError

Configuration is invalid, missing, or internally inconsistent.

Raised for bad parameter combinations and for lookups of things that were never registered.

Source code in src/oversampleqa/exceptions.py

class ConfigurationError(OversampleQAError):
    """Configuration is invalid, missing, or internally inconsistent.

    Raised for bad parameter combinations and for lookups of things that were
    never registered.
    """

MetricError

Bases: OversampleQAError

A distance metric could not be resolved or computed.

Source code in src/oversampleqa/exceptions.py

class MetricError(OversampleQAError):
    """A distance metric could not be resolved or computed."""

OversampleQAError

Bases: Exception

Base class for every error raised by OversampleQA.

Source code in src/oversampleqa/exceptions.py

class OversampleQAError(Exception):
    """Base class for every error raised by OversampleQA."""

ValidationConfig dataclass

Immutable validation configuration.

Source code in src/oversampleqa/types.py

@dataclass(frozen=True)
class ValidationConfig:
    """Immutable validation configuration."""

    hidden_ratio: float = 0.1
    metric: str = "hassanat"
    return_details: bool = False
    random_state: int | None = 42
    reference: ReferenceSet = "hidden_minority"
    n_repeats: int = 1

    def __post_init__(self) -> None:
        if not 0 < self.hidden_ratio < 1:
            raise ValueError("hidden_ratio must be between 0 and 1")

ValidationDetails dataclass

Detailed outcome of a single validation run.

Replaces the former (error_rate, n_errors, dist_hidden, dist_min) 4-tuple returned by return_details=True.

Attributes

error_rate: Fraction of synthetic points strictly closer to the hidden majority than to the minority reference set. nan when no synthetic samples were produced -- that is an absent measurement, not a perfect score. n_errors: Count behind error_rate. n_synthetic: Number of synthetic points scored. n_ties: Points exactly equidistant from both reference sets. Counted separately rather than scored as errors; a large value indicates duplicated or heavily quantised features. duplication_rate: Fraction of synthetic points coinciding with a reference point. A sampler that only duplicates scores 1.0, and its error rate carries no information about synthesis quality. reference: Which minority set the comparison used. dist_hidden, dist_min: Distance matrices from synthetic points to the hidden majority and to the minority reference set.

Source code in src/oversampleqa/types.py

@dataclass(frozen=True)
class ValidationDetails:
    """Detailed outcome of a single validation run.

    Replaces the former ``(error_rate, n_errors, dist_hidden, dist_min)``
    4-tuple returned by ``return_details=True``.

    Attributes
    ----------
    error_rate:
        Fraction of synthetic points strictly closer to the hidden majority
        than to the minority reference set. ``nan`` when no synthetic samples
        were produced -- that is an absent measurement, not a perfect score.
    n_errors:
        Count behind ``error_rate``.
    n_synthetic:
        Number of synthetic points scored.
    n_ties:
        Points exactly equidistant from both reference sets. Counted
        separately rather than scored as errors; a large value indicates
        duplicated or heavily quantised features.
    duplication_rate:
        Fraction of synthetic points coinciding with a reference point. A
        sampler that only duplicates scores 1.0, and its error rate carries
        no information about synthesis quality.
    reference:
        Which minority set the comparison used.
    dist_hidden, dist_min:
        Distance matrices from synthetic points to the hidden majority and to
        the minority reference set.
    """

    error_rate: float
    n_errors: int
    n_synthetic: int
    n_ties: int
    duplication_rate: float
    reference: ReferenceSet
    dist_hidden: FloatArray
    dist_min: FloatArray
    n_repeats: int = 1
    rates: tuple[float, ...] = ()
    mean: float = float("nan")
    std: float = float("nan")
    interval: tuple[float, float] | None = None

    @property
    def has_dispersion(self) -> bool:
        """Whether more than one hold-out split was drawn."""
        return self.n_repeats > 1

    def to_dict(self) -> dict[str, Any]:
        """Flat, JSON-safe mapping.

        ``dist_hidden`` and ``dist_min`` are deliberately excluded: they are
        working arrays of shape ``(n_synthetic, n_reference)``, often megabytes,
        and they are inputs to the summary rather than part of it. Callers that
        need them have the dataclass.
        """
        return {
            "error_rate": self.error_rate,
            "n_errors": self.n_errors,
            "n_synthetic": self.n_synthetic,
            "n_ties": self.n_ties,
            "duplication_rate": self.duplication_rate,
            "reference": self.reference,
            "n_repeats": self.n_repeats,
            "rates": list(self.rates),
            "mean": self.mean,
            "std": self.std,
            "interval": list(self.interval) if self.interval else None,
        }

has_dispersion property

Whether more than one hold-out split was drawn.

to_dict()

Flat, JSON-safe mapping.

dist_hidden and dist_min are deliberately excluded: they are working arrays of shape (n_synthetic, n_reference), often megabytes, and they are inputs to the summary rather than part of it. Callers that need them have the dataclass.

Source code in src/oversampleqa/types.py

def to_dict(self) -> dict[str, Any]:
    """Flat, JSON-safe mapping.

    ``dist_hidden`` and ``dist_min`` are deliberately excluded: they are
    working arrays of shape ``(n_synthetic, n_reference)``, often megabytes,
    and they are inputs to the summary rather than part of it. Callers that
    need them have the dataclass.
    """
    return {
        "error_rate": self.error_rate,
        "n_errors": self.n_errors,
        "n_synthetic": self.n_synthetic,
        "n_ties": self.n_ties,
        "duplication_rate": self.duplication_rate,
        "reference": self.reference,
        "n_repeats": self.n_repeats,
        "rates": list(self.rates),
        "mean": self.mean,
        "std": self.std,
        "interval": list(self.interval) if self.interval else None,
    }

ValidationError

Bases: OversampleQAError

A validation run could not produce a meaningful result.

Covers malformed input as well as data that cannot support the estimand -- for example a minority class too small to hold anything out of.

Source code in src/oversampleqa/exceptions.py

class ValidationError(OversampleQAError):
    """A validation run could not produce a meaningful result.

    Covers malformed input as well as data that cannot support the estimand --
    for example a minority class too small to hold anything out of.
    """

ValidationMode

Bases: Enum

Validation execution modes.

Source code in src/oversampleqa/types.py

class ValidationMode(Enum):
    """Validation execution modes."""

    STANDARD = "standard"
    MEMORY_EFFICIENT = "memory_efficient"
    PARALLEL = "parallel"
    ASYNC = "async"

ValidationResult

Bases: TypedDict

Typed structure for validation result.

Source code in src/oversampleqa/types.py

class ValidationResult(TypedDict, total=False):
    """Typed structure for validation result."""

    error_rate: float
    n_errors: int
    n_synthetic: int
    confidence_interval: tuple[float, float]
    metadata: dict[str, Any]

create_benchmark_report(results_df, output_path='benchmark_report.html')

Create a lightweight HTML report summarising benchmark statistics.

Parameters:

Name	Type	Description	Default
results_df	DataFrame	Benchmark results dataframe.	required
output_path	str	Output HTML path.	'benchmark_report.html'

Returns:

Type	Description
Path	Path to the generated report.

Source code in src/oversampleqa/advanced_benchmark.py

def create_benchmark_report(
    results_df: pd.DataFrame, output_path: str = "benchmark_report.html"
) -> Path:
    """Create a lightweight HTML report summarising benchmark statistics.

    Args:
        results_df: Benchmark results dataframe.
        output_path: Output HTML path.

    Returns:
        Path to the generated report.
    """

    output = Path(output_path)
    output.parent.mkdir(parents=True, exist_ok=True)

    if results_df.empty:
        html = (
            "<html><body><h1>No benchmark results available.</h1>"
            "<h2>Run metadata</h2>"
            f"{report_metadata_html(results_df)}</body></html>"
        )
        output.write_text(html, encoding="utf-8")
        write_export_metadata(output, export_kind="statistical_benchmark_report")
        return output

    summary = results_df.copy()
    summary["ci"] = summary.apply(
        lambda row: f"[{row['ci_lower']:.3f}, {row['ci_upper']:.3f}]", axis=1
    )
    summary["p_values"] = summary["pairwise_p_values"].fillna("{}")
    summary["effect_sizes"] = summary["pairwise_effect_sizes"].fillna("{}")

    table_html = summary[
        [
            "dataset_name",
            "oversampler_name",
            "metric",
            "mean_error",
            "std_error",
            "ci",
            "p_values",
            "effect_sizes",
        ]
    ].to_html(index=False, escape=False)

    html = f"""
    <html>
        <head>
            <title>OversampleQA Benchmark Report</title>
            <style>
                body {{ font-family: Arial, sans-serif; padding: 2rem; }}
                table {{ border-collapse: collapse; width: 100%; }}
                th, td {{ border: 1px solid #ccc; padding: 0.5rem; }}
                th {{ background: #f5f5f5; }}
            </style>
        </head>
        <body>
            <h1>OversampleQA Benchmark Report</h1>
            <h2>Run metadata</h2>
            {report_metadata_html(results_df)}
            <h2>Results</h2>
            {table_html}
        </body>
    </html>
    """
    output.write_text(html, encoding="utf-8")
    write_export_metadata(
        output,
        export_kind="statistical_benchmark_report",
        data=summary,
    )
    return output

format_statistical_summary(results_df, significance_level=0.05)

Render a Markdown summary of a statistical benchmark frame.

The frame is expected to come from :meth:StatisticalBenchmark.run_comprehensive_benchmark. The summary lists, per dataset, the mean error, standard deviation and confidence interval for each oversampler/metric, followed by the statistically significant pairwise comparisons (corrected p-value below significance_level).

Parameters:

Name	Type	Description	Default
results_df	DataFrame	Benchmark results dataframe.	required
significance_level	float	Threshold below which a pairwise p-value is reported.	0.05

Returns:

Type	Description
str	A Markdown-formatted string.

Source code in src/oversampleqa/advanced_benchmark.py

def format_statistical_summary(
    results_df: pd.DataFrame, significance_level: float = 0.05
) -> str:
    """Render a Markdown summary of a statistical benchmark frame.

    The frame is expected to come from
    :meth:`StatisticalBenchmark.run_comprehensive_benchmark`. The summary lists,
    per dataset, the mean error, standard deviation and confidence interval for
    each oversampler/metric, followed by the statistically significant pairwise
    comparisons (corrected p-value below ``significance_level``).

    Args:
        results_df: Benchmark results dataframe.
        significance_level: Threshold below which a pairwise p-value is reported.

    Returns:
        A Markdown-formatted string.
    """
    if results_df.empty:
        return (
            "# OversampleQA Statistical Benchmark\n\nNo benchmark results available.\n"
        )

    lines = ["# OversampleQA Statistical Benchmark", ""]
    lines.append(
        "Confidence intervals use the configured confidence level. Pairwise "
        "p-values and effect sizes (Cohen's d) compare oversamplers on the same "
        "dataset and metric, corrected by the configured method."
    )
    lines.append("")

    for dataset_name, group in results_df.groupby("dataset_name", sort=True):
        lines.append(f"## Dataset: {dataset_name}")
        lines.append("")
        lines.append("| Oversampler | Metric | Mean error | Std | CI | n |")
        lines.append("| --- | --- | --- | --- | --- | --- |")
        for _, row in group.iterrows():
            ci = f"[{row['ci_lower']:.3f}, {row['ci_upper']:.3f}]"
            lines.append(
                f"| {row['oversampler_name']} | {row['metric']} | "
                f"{row['mean_error']:.3f} | {row['std_error']:.3f} | {ci} | "
                f"{int(row['n_observations'])} |"
            )
        lines.append("")

        significant = _significant_pairwise(group, significance_level)
        if significant:
            lines.append(
                f"Significant pairwise differences (p < {significance_level}):"
            )
            for label, p_val, effect in significant:
                effect_str = f", d={effect:.2f}" if effect is not None else ""
                lines.append(f"- {label}: p={p_val:.4f}{effect_str}")
            lines.append("")

    return "\n".join(lines)

compute_ranking(results)

Rank oversamplers within each experiment, then aggregate the ranks.

Error rates are not comparable across datasets, hold-out ratios or metrics: an easy dataset scores near 0.1 and a hard one near 0.9, and hassanat scores roughly twice euclidean on the same data. Pooling them and taking a mean asks a question with no answer.

Ranking within each (dataset, hidden_ratio, metric) and averaging those ranks is the Demsar (2006) protocol, and the same logic underlying :func:~oversampleqa.inference.friedman_nemenyi -- so the ranking here and the significance test there answer the same question.

Parameters:

Name	Type	Description	Default
results	DataFrame	Long-format benchmark frame from :func:run_benchmark.	required

Returns:

Type	Description
DataFrame	Summary indexed by oversampler with mean_rank (lower is better),
DataFrame	rank, n_specifications, and the pooled mean, std and
DataFrame	n_missing retained for reference.

Warns:

Type	Description
UserWarning	If oversamplers were ranked over different numbers of experiments. Mean ranks computed over different sets are not comparable, and the imbalance is usually caused by skipped runs.

Notes

Averaging the raw error rate was not merely imprecise, it inverted results. Given a sampler that beats another on every dataset while having more of its runs skipped on the hard one, the pooled mean favours the loser -- Simpson's paradox, reachable here because the hold-out guards legitimately drop runs.

nan runs are excluded rather than counted as zero, and the count is reported in n_missing.

Source code in src/oversampleqa/benchmark.py

def compute_ranking(results: pd.DataFrame) -> pd.DataFrame:
    """Rank oversamplers within each experiment, then aggregate the ranks.

    Error rates are not comparable across datasets, hold-out ratios or metrics:
    an easy dataset scores near 0.1 and a hard one near 0.9, and hassanat scores
    roughly twice euclidean on the same data. Pooling them and taking a mean
    asks a question with no answer.

    Ranking within each ``(dataset, hidden_ratio, metric)`` and averaging those
    ranks is the Demsar (2006) protocol, and the same logic underlying
    :func:`~oversampleqa.inference.friedman_nemenyi` -- so the ranking here and
    the significance test there answer the same question.

    Args:
        results: Long-format benchmark frame from :func:`run_benchmark`.

    Returns:
        Summary indexed by oversampler with ``mean_rank`` (lower is better),
        ``rank``, ``n_specifications``, and the pooled ``mean``, ``std`` and
        ``n_missing`` retained for reference.

    Warns:
        UserWarning: If oversamplers were ranked over different numbers of
            experiments. Mean ranks computed over different sets are not
            comparable, and the imbalance is usually caused by skipped runs.

    Notes:
        Averaging the raw error rate was not merely imprecise, it inverted
        results. Given a sampler that beats another on *every* dataset while
        having more of its runs skipped on the hard one, the pooled mean
        favours the loser -- Simpson's paradox, reachable here because the
        hold-out guards legitimately drop runs.

        ``nan`` runs are excluded rather than counted as zero, and the count is
        reported in ``n_missing``.
    """
    grouped = results.groupby("oversampler")["error_rate"]
    summary = grouped.agg(
        mean=lambda s: s.mean(skipna=True),
        std=lambda s: s.std(skipna=True),
    )
    summary["n_missing"] = grouped.apply(lambda s: int(s.isna().sum()))

    spec = [c for c in _SPECIFICATION_COLUMNS if c in results.columns]
    if not spec:
        # Nothing identifies separate experiments, so every row is already
        # comparable and the pooled mean is the only available ordering.
        summary["mean_rank"] = summary["mean"].rank(method="average")
        summary["n_specifications"] = 1
        summary["rank"] = summary["mean_rank"].rank(method="min")
        return summary

    # One score per (experiment, oversampler), then rank within the experiment.
    per_spec = results.groupby([*spec, "oversampler"])["error_rate"].mean()
    ranks = per_spec.groupby(level=list(range(len(spec)))).rank(method="average")

    mean_rank = ranks.groupby("oversampler").mean()
    counts = ranks.groupby("oversampler").count()
    summary["mean_rank"] = mean_rank
    summary["n_specifications"] = counts.astype("Int64")
    summary["rank"] = summary["mean_rank"].rank(method="min")

    if counts.nunique() > 1:
        warnings.warn(
            "Oversamplers were ranked over different numbers of experiments "
            f"({counts.to_dict()}). Mean ranks computed over different sets of "
            "experiments are not comparable; the imbalance usually means some "
            "runs were skipped. Check n_missing.",
            UserWarning,
            stacklevel=2,
        )
    return summary

export_benchmark_results(results, output_path, fmt='csv')

Export benchmark summary to CSV, JSON or Markdown.

Parameters:

Name	Type	Description	Default
results	DataFrame	Benchmark results dataframe.	required
output_path	str	Destination path.	required
fmt	str	Output format: csv, json, markdown or html. All four render the same ranking frame.	'csv'

Raises:

Type	Description
ValueError	If fmt is not one of the four.

Source code in src/oversampleqa/benchmark.py

def export_benchmark_results(
    results: pd.DataFrame, output_path: str, fmt: str = "csv"
) -> None:
    """Export benchmark summary to CSV, JSON or Markdown.

    Args:
        results: Benchmark results dataframe.
        output_path: Destination path.
        fmt: Output format: ``csv``, ``json``, ``markdown`` or ``html``.
            All four render the same ranking frame.

    Raises:
        ValueError: If ``fmt`` is not one of the four.
    """
    output = pathlib.Path(output_path)
    summary = compute_ranking(results)
    summary.attrs["source"] = {
        "row_count": len(results),
        "columns": [str(column) for column in results.columns],
        "attrs": dict(results.attrs),
    }
    fmt = fmt.lower()
    if fmt == "csv":
        summary.to_csv(output)
    elif fmt == "json":
        # nan becomes null. JSON has no NaN literal, and emitting one produces a
        # document that strict parsers reject; null at least round-trips.
        write_json(output, summary.reset_index().to_dict(orient="records"))
    elif fmt == "markdown":
        # This used to be `summary.to_csv(sep="|")`, which is not Markdown: no
        # header separator row and no edge pipes, so it rendered as one run-on
        # paragraph. The same bug was fixed in report.py; it survived here
        # because the renderer was duplicated rather than shared.
        output.write_text(frame_to_markdown(summary), encoding="utf-8")
    elif fmt == "html":
        output.write_text(frame_to_html(summary), encoding="utf-8")
    else:
        raise ValueError("fmt must be 'csv', 'json', 'markdown' or 'html'")

    write_export_metadata(output, export_kind="benchmark_summary", data=summary)

load_standard_datasets(include_openml=False)

Return a list of simple synthetic datasets for benchmarking.

Parameters

include_openml: Whether to attempt downloading additional datasets from OpenML. The default is False to avoid slow network calls during tests.

Returns

list of dict Each entry contains name, data, target, minority_label and provenance keys. The provenance value is a dict describing the dataset's source, generator, params, url, license and notes.

Source code in src/oversampleqa/benchmark.py

def load_standard_datasets(include_openml: bool = False) -> list[dict]:
    """Return a list of simple synthetic datasets for benchmarking.

    Parameters
    ----------
    include_openml:
        Whether to attempt downloading additional datasets from OpenML. The
        default is ``False`` to avoid slow network calls during tests.

    Returns
    -------
    list of dict
        Each entry contains ``name``, ``data``, ``target``,
        ``minority_label`` and ``provenance`` keys. The ``provenance`` value
        is a dict describing the dataset's ``source``, ``generator``,
        ``params``, ``url``, ``license`` and ``notes``.
    """

    from sklearn.datasets import (
        make_blobs,
        make_circles,
        make_classification,
        make_moons,
    )

    datasets: list[dict] = []

    if include_openml:
        from sklearn.datasets import fetch_openml
        from sklearn.preprocessing import StandardScaler

        openml_specs = [
            ("yeast-4", "class"),
            ("yeast-5", "class"),
            ("yeast-6", "class"),
            ("vehicle", "Class"),
        ]
        for name, target_col in openml_specs:
            try:  # pragma: no cover - network dependent
                ds = fetch_openml(name, version=1, as_frame=False)
                X = StandardScaler().fit_transform(ds.data)
                y = ds[target_col].astype(int)
                minority = 1 if np.sum(y == 1) < np.sum(y == 0) else 0
                datasets.append(
                    {
                        "name": name,
                        "data": X,
                        "target": y,
                        "minority_label": minority,
                        "provenance": openml_provenance(
                            name,
                            1,
                            notes=(
                                "Downloaded from OpenML (version pinned to 1) and "
                                "standardized with StandardScaler."
                            ),
                        ),
                    }
                )
            except Exception as exc:  # pragma: no cover - network dependent
                logger.warning("Failed to fetch %s: %s", name, exc)

    Xc, yc = make_classification(n_samples=1000, weights=[0.9, 0.1], random_state=42)
    datasets.append(
        {
            "name": "classification",
            "data": Xc,
            "target": yc,
            "minority_label": 1,
            "provenance": synthetic_provenance(
                "sklearn.datasets.make_classification",
                n_samples=1000,
                weights=[0.9, 0.1],
                random_state=42,
            ),
        }
    )

    Xm, ym = make_moons(n_samples=600, noise=0.2, random_state=42)
    Xm, ym = _imbalance(Xm, ym, 1, keep=60, rng=np.random.default_rng(42))
    datasets.append(
        {
            "name": "moons",
            "data": Xm,
            "target": ym,
            "minority_label": 1,
            "provenance": synthetic_provenance(
                "sklearn.datasets.make_moons",
                n_samples=600,
                noise=0.2,
                random_state=42,
                minority_kept=60,
                subsample_seed=42,
            ),
        }
    )

    Xr, yr = make_circles(n_samples=600, noise=0.1, factor=0.5, random_state=42)
    Xr, yr = _imbalance(Xr, yr, 1, keep=60, rng=np.random.default_rng(42))
    datasets.append(
        {
            "name": "circles",
            "data": Xr,
            "target": yr,
            "minority_label": 1,
            "provenance": synthetic_provenance(
                "sklearn.datasets.make_circles",
                n_samples=600,
                noise=0.1,
                factor=0.5,
                random_state=42,
                minority_kept=60,
                subsample_seed=42,
            ),
        }
    )

    Xb, yb = make_blobs(
        n_samples=[450, 60],
        centers=[(-2, 0), (2, 0)],
        cluster_std=[1.0, 1.0],
        random_state=42,
    )
    datasets.append(
        {
            "name": "blobs",
            "data": Xb,
            "target": yb,
            "minority_label": 1,
            "provenance": synthetic_provenance(
                "sklearn.datasets.make_blobs",
                n_samples=[450, 60],
                centers=[(-2, 0), (2, 0)],
                cluster_std=[1.0, 1.0],
                random_state=42,
            ),
        }
    )

    Xh, yh = make_classification(
        n_samples=1200,
        n_features=10,
        n_informative=5,
        n_redundant=2,
        weights=[0.95, 0.05],
        class_sep=0.5,
        random_state=7,
    )
    datasets.append(
        {
            "name": "hard_classification",
            "data": Xh,
            "target": yh,
            "minority_label": 1,
            "provenance": synthetic_provenance(
                "sklearn.datasets.make_classification",
                n_samples=1200,
                n_features=10,
                n_informative=5,
                n_redundant=2,
                weights=[0.95, 0.05],
                class_sep=0.5,
                random_state=7,
            ),
        }
    )

    Xe, ye = make_classification(
        n_samples=1200,
        n_features=2,
        n_redundant=0,
        n_clusters_per_class=1,
        weights=[0.95, 0.05],
        class_sep=2.0,
        random_state=21,
    )
    datasets.append(
        {
            "name": "easy_linear",
            "data": Xe,
            "target": ye,
            "minority_label": 1,
            "provenance": synthetic_provenance(
                "sklearn.datasets.make_classification",
                n_samples=1200,
                n_features=2,
                n_redundant=0,
                n_clusters_per_class=1,
                weights=[0.95, 0.05],
                class_sep=2.0,
                random_state=21,
            ),
        }
    )

    Xo, yo = make_classification(
        n_samples=1200,
        n_features=2,
        n_redundant=0,
        n_clusters_per_class=1,
        weights=[0.95, 0.05],
        class_sep=0.3,
        flip_y=0.03,
        random_state=22,
    )
    datasets.append(
        {
            "name": "overlap_classification",
            "data": Xo,
            "target": yo,
            "minority_label": 1,
            "provenance": synthetic_provenance(
                "sklearn.datasets.make_classification",
                n_samples=1200,
                n_features=2,
                n_redundant=0,
                n_clusters_per_class=1,
                weights=[0.95, 0.05],
                class_sep=0.3,
                flip_y=0.03,
                random_state=22,
            ),
        }
    )

    return datasets

run_benchmark(datasets, oversamplers, hidden_ratios=None, n_runs=10, distance_metric='hassanat', random_state=None)

Run validation across datasets and oversampling methods.

Parameters:

Name	Type	Description	Default
datasets	list[dict]	Dataset descriptors containing data and target.	required
oversamplers	list	Oversampler instances.	required
hidden_ratios	list[float] | None	Hidden ratios to evaluate.	None
n_runs	int	Number of repetitions per configuration.	10
distance_metric	str	Distance metric name.	'hassanat'
random_state	RandomStateLike	RNG seed for reproducibility.	None

Returns:

Type	Description
DataFrame	DataFrame with per-run error rates.

Source code in src/oversampleqa/benchmark.py

def run_benchmark(
    datasets: list[dict],
    oversamplers: list,
    hidden_ratios: list[float] | None = None,
    n_runs: int = 10,
    distance_metric: str = "hassanat",
    random_state: RandomStateLike = None,
) -> pd.DataFrame:
    """Run validation across datasets and oversampling methods.

    Args:
        datasets: Dataset descriptors containing ``data`` and ``target``.
        oversamplers: Oversampler instances.
        hidden_ratios: Hidden ratios to evaluate.
        n_runs: Number of repetitions per configuration.
        distance_metric: Distance metric name.
        random_state: RNG seed for reproducibility.

    Returns:
        DataFrame with per-run error rates.
    """
    if hidden_ratios is None:
        hidden_ratios = [0.1, 0.25, 0.5]

    results = []
    rng = as_generator(random_state)

    logger.info("Starting benchmark with %d datasets", len(datasets))

    for data in datasets:
        X, y = data["data"], data["target"]
        minority_label = data.get("minority_label", 1)
        for oversampler in oversamplers:
            for ratio in hidden_ratios:
                for run in range(n_runs):
                    rs = rng.integers(0, 1_000_000)
                    oversampler.random_state = rs
                    # Vary the hold-out split per run as well. Reseeding only the
                    # oversampler left every run sharing one split, so the spread
                    # across runs omitted the largest source of variance.
                    split_seed = int(rng.integers(0, 2**31 - 1))
                    try:
                        error = validate_oversampling(
                            X,
                            y,
                            minority_label,
                            oversampler,
                            hidden_ratio=ratio,
                            metric=distance_metric,
                            random_state=split_seed,
                        )
                    except ValueError as exc:
                        # A dataset whose minority is too small to hold out from
                        # cannot support the estimand. Record it as a missing
                        # measurement and carry on, rather than aborting the whole
                        # sweep or -- worse -- recording a 0.0 that would read as a
                        # perfect score. compute_ranking reports these as n_missing.
                        warnings.warn(
                            f"Skipping {data.get('name', 'dataset')} with "
                            f"{oversampler.__class__.__name__} at hidden_ratio="
                            f"{ratio}: {exc}",
                            UserWarning,
                            stacklevel=2,
                        )
                        error = float("nan")
                    except Exception:
                        logger.exception("Validation failed for %s", oversampler)
                        raise
                    results.append(
                        {
                            "dataset": data.get("name", "dataset"),
                            "oversampler": oversampler.__class__.__name__,
                            # The metric is part of what identifies a
                            # measurement, not just an argument to it. Without
                            # it, concatenating two sweeps run under different
                            # metrics gives a frame whose rows cannot be told
                            # apart -- and error rates are not comparable
                            # across metrics.
                            "metric": distance_metric,
                            "hidden_ratio": ratio,
                            "run": run,
                            "split_seed": split_seed,
                            "oversampler_random_state": int(rs),
                            "minority_label": minority_label,
                            "reference": "hidden_minority",
                            "oversampleqa_version": _PACKAGE_VERSION,
                            "error_rate": error,
                        }
                    )
    # Fixed column order even when empty, so a caller that correctly handles
    # "no results" still gets a frame it can select columns from.
    frame = pd.DataFrame(results, columns=list(_BENCHMARK_COLUMNS))
    frame.attrs["dataset_provenance"] = {
        str(data.get("name", "dataset")): data["provenance"]
        for data in datasets
        if "provenance" in data
    }
    frame.attrs["benchmark_parameters"] = {
        "hidden_ratios": hidden_ratios,
        "n_runs": n_runs,
        "distance_metric": distance_metric,
        "random_state": repr(random_state),
    }
    return frame

cli_main()

Run the CLI validation workflow.

This entry point loads the dataset, configures the oversampler, runs the validation, and optionally writes a report or plot.

Source code in src/oversampleqa/cli.py

def main() -> None:
    """Run the CLI validation workflow.

    This entry point loads the dataset, configures the oversampler, runs the
    validation, and optionally writes a report or plot.
    """
    logging.basicConfig(level=logging.INFO)
    args = parse_args()
    try:
        df = pd.read_csv(args.csv)
    except Exception as exc:  # pragma: no cover - runtime guard
        logger.exception("Failed to read CSV: %s", exc)
        raise
    if args.target not in df.columns:
        raise ValueError(f"Target column '{args.target}' not found in CSV")
    X = df.drop(columns=[args.target]).values
    y = df[args.target].values

    mod = import_module("imblearn.over_sampling")
    oversampler_cls = getattr(mod, args.oversampler)
    oversampler = oversampler_cls()

    try:
        error = validate_oversampling(
            X,
            y,
            minority_label=args.minority_label,
            oversampler=oversampler,
            hidden_ratio=args.hidden_ratio,
            metric=args.distance,
        )
    except Exception:
        logger.exception("Validation failed")
        raise
    print(f"Error rate: {error:.3f}")

    if args.out:
        with open(args.out, "w", encoding="utf-8") as f:
            f.write(f"Error rate: {error:.3f}\n")

    if args.plot:
        # Refit oversampler on the full dataset for visualization
        vis_os = oversampler_cls()
        X_res, y_res = vis_os.fit_resample(X, y)
        mask = y == args.minority_label
        minority = X[mask]
        majority = X[~mask]
        synthetic = extract_synthetic_samples(X, X_res, y_res, args.minority_label)
        plot_sample_distribution(majority, minority, synthetic, save_path=args.plot)

cluster_based_diagnostics(majority, synthetic, n_clusters=5, algorithm='kmeans', eps=0.5, min_samples=5, random_state=None)

Flag synthetic samples that fall in majority-dominated clusters.

Parameters

majority, synthetic : ndarray Arrays of majority and synthetic samples with shape (n_samples, n_features). n_clusters : int, default=5 Number of clusters for the k-means algorithm. algorithm : {"kmeans", "dbscan"}, default="kmeans" Clustering algorithm to use. eps : float, default=0.5 Neighborhood radius when using DBSCAN. min_samples : int, default=5 Minimum samples per cluster for DBSCAN. random_state : int, optional Random state for k-means.

Returns

flagged : ndarray of bool Boolean mask indicating which synthetic samples are located in clusters dominated by majority data. overlap_score : float Silhouette score of the clustering which acts as a crude overlap metric.

Source code in src/oversampleqa/clustering.py

def cluster_based_diagnostics(
    majority: NDArray[np.floating],
    synthetic: NDArray[np.floating],
    n_clusters: int = 5,
    algorithm: str = "kmeans",
    eps: float = 0.5,
    min_samples: int = 5,
    random_state: int | None = None,
) -> tuple[NDArray[np.bool_], float]:
    """Flag synthetic samples that fall in majority-dominated clusters.

    Parameters
    ----------
    majority, synthetic : ndarray
        Arrays of majority and synthetic samples with shape ``(n_samples, n_features)``.
    n_clusters : int, default=5
        Number of clusters for the k-means algorithm.
    algorithm : {"kmeans", "dbscan"}, default="kmeans"
        Clustering algorithm to use.
    eps : float, default=0.5
        Neighborhood radius when using DBSCAN.
    min_samples : int, default=5
        Minimum samples per cluster for DBSCAN.
    random_state : int, optional
        Random state for k-means.

    Returns
    -------
    flagged : ndarray of bool
        Boolean mask indicating which synthetic samples are located in clusters
        dominated by majority data.
    overlap_score : float
        Silhouette score of the clustering which acts as a crude overlap metric.
    """

    if len(synthetic) == 0:
        return np.array([], dtype=bool), 0.0

    from sklearn.cluster import DBSCAN, KMeans
    from sklearn.metrics import silhouette_score

    X = np.vstack([majority, synthetic])

    try:
        if algorithm == "kmeans":
            labels = KMeans(
                n_clusters=n_clusters, random_state=random_state
            ).fit_predict(X)
        elif algorithm == "dbscan":
            labels = DBSCAN(eps=eps, min_samples=min_samples).fit_predict(X)
        else:
            raise ValueError("algorithm must be 'kmeans' or 'dbscan'")
    except Exception:  # pragma: no cover - defensive
        logger.exception("Clustering failed")
        raise

    maj_labels = labels[: len(majority)]
    synth_labels = labels[len(majority) :]

    flagged = np.zeros(len(synthetic), dtype=bool)
    unique_labels = [lbl for lbl in np.unique(labels) if lbl != -1]

    for lbl in unique_labels:
        maj_mask = maj_labels == lbl
        synth_mask = synth_labels == lbl
        n_maj = maj_mask.sum()
        n_syn = synth_mask.sum()
        if n_syn == 0:
            continue
        ratio = n_maj / (n_maj + n_syn)
        if ratio > 0.5:
            flagged[synth_mask] = True

    if len(np.unique(labels)) > 1:
        try:
            score = silhouette_score(X, labels)
        except Exception:  # pragma: no cover - defensive
            logger.exception("Failed to compute silhouette score")
            score = 0.0
    else:
        score = 0.0

    return flagged, float(score)

deprecated(*, removal_version, replacement=None, reason=None, category=DeprecationWarning)

Mark a function, method or class as deprecated.

The emitted warning names the replacement and the removal version, which is what :doc:/api_stability promises and what a caller needs in order to act. A note is appended to the docstring so the deprecation is visible in the rendered documentation as well as at runtime.

The warning is raised with stacklevel pointing at the caller, not at this wrapper. This matters more than it looks: Python's default filters hide DeprecationWarning unless it originates in __main__, and per-module filters key on the reported location. A warning that reports itself as coming from inside oversampleqa is invisible to exactly the people who need to see it.

Parameters:

Name	Type	Description	Default
removal_version	str	Release in which the name disappears, e.g. "0.6.0". Required -- a deprecation without a deadline is a permanent warning.	required
replacement	str | None	What to use instead, if there is a direct successor.	None
reason	str | None	Extra context appended to the message, for cases where the replacement is not a simple substitution.	None
category	type[Warning]	Warning class. Defaults to DeprecationWarning. Use FutureWarning when the change alters results rather than spelling, since that one is shown to end users by default.	DeprecationWarning

Returns:

Type	Description
Callable[[F], F]	A decorator that wraps the target, preserving its metadata.

Example

@deprecated(removal_version="0.6.0", replacement="new_name") ... def old_name() -> int: ... return 1 import warnings with warnings.catch_warnings(record=True) as caught: ... warnings.simplefilter("always") ... old_name() ... str(caught[0].message) 1 'old_name is deprecated and will be removed in 0.6.0. Use new_name instead.'

Source code in src/oversampleqa/deprecation.py

def deprecated(
    *,
    removal_version: str,
    replacement: str | None = None,
    reason: str | None = None,
    category: type[Warning] = DeprecationWarning,
) -> Callable[[F], F]:
    """Mark a function, method or class as deprecated.

    The emitted warning names the replacement and the removal version, which is
    what :doc:`/api_stability` promises and what a caller needs in order to act.
    A note is appended to the docstring so the deprecation is visible in the
    rendered documentation as well as at runtime.

    The warning is raised with ``stacklevel`` pointing at the **caller**, not at
    this wrapper. This matters more than it looks: Python's default filters hide
    ``DeprecationWarning`` unless it originates in ``__main__``, and per-module
    filters key on the reported location. A warning that reports itself as
    coming from inside oversampleqa is invisible to exactly the people who need
    to see it.

    Args:
        removal_version: Release in which the name disappears, e.g. ``"0.6.0"``.
            Required -- a deprecation without a deadline is a permanent warning.
        replacement: What to use instead, if there is a direct successor.
        reason: Extra context appended to the message, for cases where the
            replacement is not a simple substitution.
        category: Warning class. Defaults to ``DeprecationWarning``. Use
            ``FutureWarning`` when the change alters results rather than
            spelling, since that one is shown to end users by default.

    Returns:
        A decorator that wraps the target, preserving its metadata.

    Example:
        >>> @deprecated(removal_version="0.6.0", replacement="new_name")
        ... def old_name() -> int:
        ...     return 1
        >>> import warnings
        >>> with warnings.catch_warnings(record=True) as caught:
        ...     warnings.simplefilter("always")
        ...     old_name()
        ...     str(caught[0].message)
        1
        'old_name is deprecated and will be removed in 0.6.0. Use new_name instead.'
    """

    def decorate(target: F) -> F:
        message = _build_message(
            getattr(target, "__name__", str(target)),
            replacement,
            removal_version,
            reason,
        )
        note = f"\n\n.. deprecated:: {removal_version}\n   {message}\n"

        if isinstance(target, type):
            # Wrap __init__ so the warning fires at the instantiation site.
            # Wrapping the class in a function instead would break isinstance,
            # subclassing and the repr.
            #
            # Rebinding a dunder on a class object is exactly the kind of thing
            # a type checker is right to distrust in general, so the surgery is
            # done through an explicitly untyped alias rather than scattered
            # per-line suppressions.
            klass: Any = target
            original_init = klass.__init__

            @functools.wraps(original_init)
            def init_wrapper(self: Any, *args: Any, **kwargs: Any) -> None:
                warnings.warn(message, category, stacklevel=2)
                original_init(self, *args, **kwargs)

            klass.__init__ = init_wrapper
            klass.__doc__ = (klass.__doc__ or "") + note
            return cast(F, klass)

        @functools.wraps(target)
        def wrapper(*args: Any, **kwargs: Any) -> Any:
            warnings.warn(message, category, stacklevel=2)
            return target(*args, **kwargs)

        wrapper.__doc__ = (target.__doc__ or "") + note
        return wrapper  # type: ignore[return-value]

    return decorate

braycurtis_distance(x1, x2)

Compute Bray-Curtis distance between two vectors.

Often used in ecology and environmental science.

Source code in src/oversampleqa/extended_distances.py

def braycurtis_distance(
    x1: NDArray[np.floating], x2: NDArray[np.floating]
) -> float:
    """Compute Bray-Curtis distance between two vectors.

    Often used in ecology and environmental science.
    """
    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")

    if np.any(x1 < 0) or np.any(x2 < 0):
        raise ValueError("Bray-Curtis distance requires non-negative inputs")

    numerator = np.sum(np.abs(x1 - x2))
    denominator = np.sum(np.abs(x1 + x2))

    if denominator == 0:
        # Sound only because the inputs are non-negative: the sum of absolute
        # values is then zero exactly when both vectors are all-zero, and the
        # distance between them really is zero. Allow a negative through and
        # the terms cancel instead -- d([-1, 0], [1, 0]) came out as 0.0, two
        # distinct points at distance zero, which is the identity-of-
        # indiscernibles violation check_metric_axioms exists to catch.
        return 0.0

    ratio: float = numerator / denominator
    return ratio

canberra_distance(x1, x2)

Compute Canberra distance between two vectors.

Canberra distance is a weighted version of Manhattan distance, useful when dealing with features of different scales.

Source code in src/oversampleqa/extended_distances.py

def canberra_distance(x1: NDArray[np.floating], x2: NDArray[np.floating]) -> float:
    """Compute Canberra distance between two vectors.

    Canberra distance is a weighted version of Manhattan distance,
    useful when dealing with features of different scales.
    """
    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")

    numerator = np.abs(x1 - x2)
    denominator = np.abs(x1) + np.abs(x2)

    # Handle division by zero
    with np.errstate(divide="ignore", invalid="ignore"):
        ratio = np.where(denominator == 0, 0.0, numerator / denominator)

    return float(np.sum(ratio))

chebyshev_distance(x1, x2)

Compute Chebyshev (L-infinity) distance between two vectors.

This is the maximum absolute difference across all dimensions.

Source code in src/oversampleqa/extended_distances.py

def chebyshev_distance(
    x1: NDArray[np.floating], x2: NDArray[np.floating]
) -> float:
    """Compute Chebyshev (L-infinity) distance between two vectors.

    This is the maximum absolute difference across all dimensions.
    """
    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")

    return float(np.max(np.abs(x1 - x2)))

correlation_distance(x1, x2)

Compute correlation distance between two vectors.

Correlation distance = 1 - Pearson correlation coefficient

Source code in src/oversampleqa/extended_distances.py

def correlation_distance(
    x1: NDArray[np.floating], x2: NDArray[np.floating]
) -> float:
    """Compute correlation distance between two vectors.

    Correlation distance = 1 - Pearson correlation coefficient
    """
    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")

    if len(x1) < 2:
        # Correlation needs at least two components to have any variance.
        raise ValueError(
            "correlation distance is undefined for vectors of length < 2: "
            "there is no variance to correlate."
        )

    with np.errstate(invalid="ignore", divide="ignore"):
        # A constant vector makes corrcoef divide by a zero standard deviation.
        # That is the case handled immediately below, so the warning is noise.
        corr_coef = np.corrcoef(x1, x2)[0, 1]

    if np.isnan(corr_coef):
        # A constant vector has zero variance, so the correlation is undefined.
        # This returned 0.0 -- "perfectly correlated" -- which made a constant
        # vector distance-zero from every other vector. METRIC_DOMAINS has
        # documented the case as undefined all along; the code disagreed.
        raise ValueError(
            "correlation distance is undefined when either vector is constant: "
            "zero variance leaves nothing to correlate. Drop constant features "
            "or rows, or use a metric defined on them such as 'euclidean'."
        )

    coefficient: float = corr_coef
    return float(np.clip(1.0 - coefficient, 0.0, 2.0))

distance_matrix(X1, X2, metric='hassanat', *, batch_size='auto', cache=None, **metric_kwargs)

Compute pairwise distance matrix using the given metric.

Parameters

X1, X2 : ndarray Input matrices containing observations. metric : str, default="hassanat" Identifier of the distance metric to use. batch_size : int or {"auto", "stream"}, default="auto" Controls batching strategy. "auto" selects a batch size that fits memory_limit_gb of :class:OptimizedDistanceMatrix. "stream" forces row-wise streaming when memory is constrained. cache : ValidationCache, optional Opt-in cache. Caching is off by default: nothing is written to disk and no directory is created unless you supply one. Worth it for expensive metrics such as hassanat; a net loss for euclidean, where hashing the inputs costs more than recomputing the result. **metric_kwargs : Additional keyword arguments are forwarded to the metric function. This enables configuration of metrics that require extra parameters, such as the inverse covariance matrix for Mahalanobis distance.

Returns

ndarray Distance matrix. When cache is supplied the array is read-only; call .copy() before modifying it.

Source code in src/oversampleqa/distance.py

def distance_matrix(
    X1: NDArray[np.floating],
    X2: NDArray[np.floating],
    metric: str = "hassanat",
    *,
    batch_size: int | str = "auto",
    cache: ValidationCache | None = None,
    **metric_kwargs: Any,
) -> NDArray[np.floating]:
    """Compute pairwise distance matrix using the given metric.

    Parameters
    ----------
    X1, X2 : ndarray
        Input matrices containing observations.
    metric : str, default="hassanat"
        Identifier of the distance metric to use.
    batch_size : int or {"auto", "stream"}, default="auto"
        Controls batching strategy. ``"auto"`` selects a batch size that fits
        ``memory_limit_gb`` of :class:`OptimizedDistanceMatrix`. ``"stream"``
        forces row-wise streaming when memory is constrained.
    cache : ValidationCache, optional
        Opt-in cache. Caching is off by default: nothing is written to disk and
        no directory is created unless you supply one. Worth it for expensive
        metrics such as ``hassanat``; a net loss for ``euclidean``, where
        hashing the inputs costs more than recomputing the result.
    **metric_kwargs :
        Additional keyword arguments are forwarded to the metric function. This
        enables configuration of metrics that require extra parameters, such as
        the inverse covariance matrix for Mahalanobis distance.

    Returns
    -------
    ndarray
        Distance matrix. When ``cache`` is supplied the array is **read-only**;
        call ``.copy()`` before modifying it.
    """
    plugin = resolve_metric(metric)
    registry = _METRICS if plugin is None else {**_METRICS, metric: plugin}
    metric_kwargs = metric_kwargs or {}
    optimizer = (
        _OPTIMIZER
        if cache is None and plugin is None
        else OptimizedDistanceMatrix(metric_registry=registry, cache=cache)
    )
    return optimizer.compute_distance_matrix(
        X1,
        X2,
        metric=metric,
        batch_size=batch_size,
        **metric_kwargs,
    )

energy_distance(x1, x2)

Compute energy distance between two 1D or 2D vectors.

The implementation follows the definition from energy statistics.

.. warning::

This is a sample-based metric, not a point metric. A 1-D input is reshaped to (len(x), 1) and treated as a set of scalar observations, not as one point in len(x)-dimensional feature space. It therefore does not measure the same kind of quantity as euclidean or hassanat, even though it is reachable through the same registry. Use it to compare two samples, not two points.

Source code in src/oversampleqa/extended_distances.py

def energy_distance(x1: NDArray[np.floating], x2: NDArray[np.floating]) -> float:
    """Compute energy distance between two 1D or 2D vectors.

    The implementation follows the definition from energy statistics.

    .. warning::

       This is a **sample-based** metric, not a point metric. A 1-D input is
       reshaped to ``(len(x), 1)`` and treated as a *set of scalar
       observations*, not as one point in ``len(x)``-dimensional feature
       space. It therefore does not measure the same kind of quantity as
       ``euclidean`` or ``hassanat``, even though it is reachable through the
       same registry. Use it to compare two samples, not two points.
    """
    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)

    x1 = x1.reshape(len(x1), -1)
    x2 = x2.reshape(len(x2), -1)

    diff_cross = np.linalg.norm(x1[:, None, :] - x2[None, :, :], axis=-1)
    term_a = diff_cross.mean()

    term_b: float
    if len(x1) > 1:
        diff_x1 = np.linalg.norm(x1[:, None, :] - x1[None, :, :], axis=-1)
        term_b = float(diff_x1[np.triu_indices(len(x1), 1)].mean())
    else:
        term_b = 0.0

    term_c: float
    if len(x2) > 1:
        diff_x2 = np.linalg.norm(x2[:, None, :] - x2[None, :, :], axis=-1)
        term_c = float(diff_x2[np.triu_indices(len(x2), 1)].mean())
    else:
        term_c = 0.0

    return float(2.0 * term_a - term_b - term_c)

hamming_distance(x1, x2)

Compute Hamming distance between two vectors.

Counts the number of positions where elements differ. Useful for categorical or binary features.

Source code in src/oversampleqa/extended_distances.py

def hamming_distance(x1: NDArray[np.generic], x2: NDArray[np.generic]) -> float:
    """Compute Hamming distance between two vectors.

    Counts the number of positions where elements differ.
    Useful for categorical or binary features.
    """
    x1 = np.asarray(x1)
    x2 = np.asarray(x2)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")

    return float(np.sum(x1 != x2))

hassanat_distance(x1, x2)

Compute the Hassanat distance between two vectors.

For each dimension :math:i, with :math:m = \min(a_i, b_i) and :math:M = \max(a_i, b_i):

.. math::

D(a_i, b_i) = \begin{cases} 1 - \dfrac{1 + m}{1 + M} & m \ge 0 \[2ex] 1 - \dfrac{1 + m + |m|}{1 + M + |m|} & m < 0 \end{cases}

and :math:HD(a, b) = \sum_i D(a_i, b_i).

Every per-dimension term lies in :math:[0, 1), which is what makes the metric invariant to feature scale and robust to outliers: no single dimension can contribute more than 1 regardless of its magnitude.

Parameters

x1, x2 : NDArray[np.floating] Input vectors of identical shape.

Returns

float Hassanat distance, in [0, n_features).

Raises

ValueError If the two vectors do not have the same shape.

References

Hassanat, A. B. (2014). Dimensionality invariant similarity measure. Journal of American Science, 10(8).

Source code in src/oversampleqa/distance.py

def hassanat_distance(x1: NDArray[np.floating], x2: NDArray[np.floating]) -> float:
    r"""Compute the Hassanat distance between two vectors.

    For each dimension :math:`i`, with :math:`m = \min(a_i, b_i)` and
    :math:`M = \max(a_i, b_i)`:

    .. math::

       D(a_i, b_i) = \begin{cases}
         1 - \dfrac{1 + m}{1 + M} & m \ge 0 \\[2ex]
         1 - \dfrac{1 + m + |m|}{1 + M + |m|} & m < 0
       \end{cases}

    and :math:`HD(a, b) = \sum_i D(a_i, b_i)`.

    Every per-dimension term lies in :math:`[0, 1)`, which is what makes the
    metric invariant to feature scale and robust to outliers: no single
    dimension can contribute more than 1 regardless of its magnitude.

    Parameters
    ----------
    x1, x2 : NDArray[np.floating]
        Input vectors of identical shape.

    Returns
    -------
    float
        Hassanat distance, in ``[0, n_features)``.

    Raises
    ------
    ValueError
        If the two vectors do not have the same shape.

    References
    ----------
    Hassanat, A. B. (2014). Dimensionality invariant similarity measure.
    *Journal of American Science*, 10(8).
    """
    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")

    mn = np.minimum(x1, x2)
    mx = np.maximum(x1, x2)
    # Adding |min| on the negative branch shifts both terms up so the ratio
    # stays in (0, 1]. The denominator is 1 + mx + shift; since mx >= mn and
    # shift = max(-mn, 0), we have mx + shift >= mn + shift >= 0, so the
    # denominator is >= 1 and can never vanish. No division guard is needed.
    shift = np.where(mn < 0.0, -mn, 0.0)
    return float(np.sum(1.0 - (1.0 + mn + shift) / (1.0 + mx + shift)))

hellinger_distance(x1, x2)

Compute the Hellinger distance between two probability vectors.

The input vectors are normalized to sum to 1 and must contain non-negative values. The distance is bounded between 0 and 1.

Source code in src/oversampleqa/extended_distances.py

def hellinger_distance(x1: NDArray[np.floating], x2: NDArray[np.floating]) -> float:
    """Compute the Hellinger distance between two probability vectors.

    The input vectors are normalized to sum to ``1`` and must contain
    non-negative values. The distance is bounded between ``0`` and ``1``.
    """

    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")
    if np.any(x1 < 0) or np.any(x2 < 0):
        raise ValueError("Hellinger distance requires non-negative inputs")

    p = x1 / x1.sum() if x1.sum() != 0 else np.zeros_like(x1)
    q = x2 / x2.sum() if x2.sum() != 0 else np.zeros_like(x2)

    return float(np.linalg.norm(np.sqrt(p) - np.sqrt(q)) / np.sqrt(2.0))

jaccard_distance(x1, x2)

Compute Jaccard distance between two binary vectors.

Jaccard distance = 1 - Jaccard similarity where Jaccard similarity = :math:|intersection| / |union|

Source code in src/oversampleqa/extended_distances.py

def jaccard_distance(x1: NDArray[np.generic], x2: NDArray[np.generic]) -> float:
    """Compute Jaccard distance between two binary vectors.

    Jaccard distance = 1 - Jaccard similarity
    where Jaccard similarity = :math:`|intersection| / |union|`
    """
    x1_raw = np.asarray(x1)
    x2_raw = np.asarray(x2)
    x1_bool = x1_raw.astype(bool)
    x2_bool = x2_raw.astype(bool)
    if x1_bool.shape != x2_bool.shape:
        raise ValueError("Input vectors must have the same shape")
    if not _is_binary(x1_raw) or not _is_binary(x2_raw):
        # Without this, casting to bool made every non-zero identical:
        # d([1.0, 3.0], [7.0, 0.2]) was 0.0, two distinct points at distance
        # zero. `boolean` is the domain this metric declares in METRIC_DOMAINS.
        raise ValueError(
            "Jaccard distance requires binary inputs: values must be 0 or 1, "
            "or a boolean array. Casting other values to bool treats every "
            "non-zero as identical, so distinct points come out at distance "
            "zero. Binarise the features first, choosing the threshold "
            "deliberately."
        )

    intersection = np.sum(x1_bool & x2_bool)
    union = np.sum(x1_bool | x2_bool)

    if union == 0:
        return 0.0  # Both vectors are all zeros

    similarity: float = intersection / union
    return 1.0 - similarity

jensen_shannon_distance(x1, x2)

Compute the Jensen-Shannon distance between two probability vectors.

The Jensen-Shannon distance is the square root of the Jensen-Shannon divergence and is symmetric and bounded between 0 and sqrt(log(2)) when using natural logarithms.

Source code in src/oversampleqa/extended_distances.py

def jensen_shannon_distance(
    x1: NDArray[np.floating], x2: NDArray[np.floating]
) -> float:
    """Compute the Jensen-Shannon distance between two probability vectors.

    The Jensen-Shannon distance is the square root of the
    Jensen-Shannon divergence and is symmetric and bounded between ``0`` and
    ``sqrt(log(2))`` when using natural logarithms.
    """

    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")
    if np.any(x1 < 0) or np.any(x2 < 0):
        raise ValueError("Jensen-Shannon distance requires non-negative inputs")

    p = x1 / x1.sum() if x1.sum() != 0 else np.zeros_like(x1)
    q = x2 / x2.sum() if x2.sum() != 0 else np.zeros_like(x2)
    m = 0.5 * (p + q)

    def _kl_div(a: NDArray[np.floating], b: NDArray[np.floating]) -> float:
        with np.errstate(divide="ignore", invalid="ignore"):
            ratio = np.where(a == 0, 1.0, a / b)
            log_term = np.log(ratio)
        return float(np.sum(np.where(a == 0, 0.0, a * log_term)))

    js_div = 0.5 * _kl_div(p, m) + 0.5 * _kl_div(q, m)
    return float(np.sqrt(js_div))

mahalanobis_distance(x1, x2, cov_inv=None)

Compute Mahalanobis distance between two vectors.

Parameters

x1, x2 : np.ndarray Input vectors cov_inv : np.ndarray Inverse covariance matrix. Required, and must be symmetric positive semi-definite -- that is what makes the result a distance. It is not validated as such on every call, because an eigenvalue check per pair would cost more than the distance itself; a negative squared distance is caught instead, which is how a non-PSD matrix usually shows up.

Note the residual case: a matrix that is not PSD can still return 0
for two distinct points, and no per-pair check can detect that. If you
build ``cov_inv`` by any route other than inverting a sample
covariance, check it once with ``np.linalg.eigvalsh``.

Returns

float Mahalanobis distance

Raises

ValueError If cov_inv is omitted, or if it yields a negative squared distance.

Source code in src/oversampleqa/extended_distances.py

def mahalanobis_distance(
    x1: NDArray[np.floating],
    x2: NDArray[np.floating],
    cov_inv: NDArray[np.floating] | None = None,
) -> float:
    """Compute Mahalanobis distance between two vectors.

    Parameters
    ----------
    x1, x2 : np.ndarray
        Input vectors
    cov_inv : np.ndarray
        Inverse covariance matrix. Required, and must be symmetric positive
        semi-definite -- that is what makes the result a distance. It is not
        validated as such on every call, because an eigenvalue check per pair
        would cost more than the distance itself; a negative squared distance
        is caught instead, which is how a non-PSD matrix usually shows up.

        Note the residual case: a matrix that is not PSD can still return 0
        for two distinct points, and no per-pair check can detect that. If you
        build ``cov_inv`` by any route other than inverting a sample
        covariance, check it once with ``np.linalg.eigvalsh``.

    Returns
    -------
    float
        Mahalanobis distance

    Raises
    ------
    ValueError
        If ``cov_inv`` is omitted, or if it yields a negative squared distance.
    """
    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")

    diff = x1 - x2

    if cov_inv is None:
        # Silently returning Euclidean relabels one metric as another. It was
        # doing exactly that in the advanced benchmark's default metric list,
        # where every "mahalanobis" row was a byte-identical copy of the
        # "euclidean" row -- double-weighting euclidean in the rankings and
        # making the pairwise correction treat one comparison as two.
        raise ValueError(
            "mahalanobis requires cov_inv: Mahalanobis distance with an "
            "identity covariance is Euclidean distance, so defaulting to it "
            "would report one metric under another's name. Estimate the "
            "inverse from the reference data, e.g. "
            "cov_inv=np.linalg.pinv(np.cov(X, rowvar=False)), and pass it "
            "through metric_kwargs."
        )

    squared = float(np.dot(diff, np.dot(cov_inv, diff)))
    if squared < 0.0:
        # A genuine inverse covariance is positive semi-definite, so this
        # quadratic form cannot be negative. When it is, np.sqrt returns nan
        # with nothing but a bare "invalid value encountered in sqrt" to say
        # why -- a warning users routinely filter, pointing at a line inside
        # this library rather than at the matrix they passed.
        #
        # A near-singular inverse can produce a tiny negative through rounding
        # alone, which is noise rather than an error, so that is clamped.
        # Anything larger means cov_inv is not an inverse covariance.
        tolerance = 1e-12 * max(1.0, float(np.dot(diff, diff)))
        if squared < -tolerance:
            raise ValueError(
                "mahalanobis requires a positive semi-definite cov_inv: the "
                f"squared distance came out negative ({squared:.6g}), which "
                "np.sqrt reports as nan. Passing the covariance itself rather "
                "than its inverse, or inverting a covariance estimated from "
                "fewer samples than features, both produce a matrix that is "
                "not. Estimate it with "
                "cov_inv=np.linalg.pinv(np.cov(X, rowvar=False))."
            )
        squared = 0.0

    return float(np.sqrt(squared))

minkowski_distance(x1, x2, p=3.0)

Compute Minkowski distance between two vectors.

Parameters

x1, x2 : np.ndarray Input vectors of same shape p : float, default=3.0 Order of the norm (p >= 1). np.inf is accepted and gives the Chebyshev distance, which is the limit as p grows.

Returns

float Minkowski distance

Raises

ValueError If the shapes differ, or p < 1.

Source code in src/oversampleqa/extended_distances.py

def minkowski_distance(
    x1: NDArray[np.floating], x2: NDArray[np.floating], p: float = 3.0
) -> float:
    """Compute Minkowski distance between two vectors.

    Parameters
    ----------
    x1, x2 : np.ndarray
        Input vectors of same shape
    p : float, default=3.0
        Order of the norm (``p >= 1``). ``np.inf`` is accepted and gives the
        Chebyshev distance, which is the limit as p grows.

    Returns
    -------
    float
        Minkowski distance

    Raises
    ------
    ValueError
        If the shapes differ, or ``p < 1``.
    """
    x1 = np.asarray(x1, dtype=float)
    x2 = np.asarray(x2, dtype=float)
    if x1.shape != x2.shape:
        raise ValueError("Input vectors must have the same shape")
    if p < 1:
        raise ValueError("p must be >= 1")

    diff = np.abs(x1 - x2)
    largest = float(diff.max()) if diff.size else 0.0
    if largest == 0.0:
        return 0.0

    if np.isinf(p):
        # The p -> infinity limit is the Chebyshev distance. The general
        # formula cannot produce it: every |d| > 1 raised to inf is inf, the
        # sum is inf, and inf ** (1 / inf) is inf ** 0, which is 1.0. So
        # `p=inf` returned 1.0 for any input at all, regardless of the data.
        return largest

    # The largest term is factored out before exponentiating. Computed
    # directly, `diff ** p` overflows at moderate p -- p=1000 gives inf, as it
    # does in scipy -- when the answer is simply the largest term. Scaling
    # every term into [0, 1] first makes the sum well behaved, and the result
    # is identical for the p values that never overflowed.
    scaled = diff / largest
    return float(largest * np.sum(scaled**p) ** (1 / p))

wasserstein_1d_distance(x1, x2)

Compute the 1D Wasserstein distance between two empirical distributions.

.. warning::

This is a sample-based metric, not a point metric. The input vector is flattened and treated as a set of scalar observations drawn from a distribution, not as one point in feature space. It therefore does not measure the same kind of quantity as euclidean or hassanat, even though it is reachable through the same registry. Use it to compare two samples, not two points.

Parameters:

Name	Type	Description	Default
x1	NDArray[floating]	Samples from distribution 1.	required
x2	NDArray[floating]	Samples from distribution 2.	required

Returns:

Type	Description
float	Wasserstein distance.

Source code in src/oversampleqa/extended_distances.py

def wasserstein_1d_distance(
    x1: NDArray[np.floating], x2: NDArray[np.floating]
) -> float:
    """Compute the 1D Wasserstein distance between two empirical distributions.

    .. warning::

       This is a **sample-based** metric, not a point metric. The input vector
       is flattened and treated as a *set of scalar observations* drawn from a
       distribution, not as one point in feature space. It therefore does not
       measure the same kind of quantity as ``euclidean`` or ``hassanat``,
       even though it is reachable through the same registry. Use it to
       compare two samples, not two points.

    Args:
        x1: Samples from distribution 1.
        x2: Samples from distribution 2.

    Returns:
        Wasserstein distance.
    """
    x1 = np.sort(np.asarray(x1, dtype=float).ravel())
    x2 = np.sort(np.asarray(x2, dtype=float).ravel())

    n = len(x1)
    m = len(x2)
    if n == 0 or m == 0:
        return 0.0

    i = j = 0
    cdf1 = cdf2 = 0.0
    last_x = min(x1[0], x2[0])
    dist = 0.0

    # W1 = integral of |F1(t) - F2(t)| dt. Between consecutive sorted points the
    # two CDFs are flat, so each interval [last_x, x) contributes
    # |F1 - F2| * (x - last_x) using the CDF values that hold *across* it --
    # that is, the values before the jump at x.
    #
    # The previous version advanced the CDF first and then added, crediting each
    # interval with the value from after its right-hand jump. On [0, 1] vs
    # [0, 3] that returned 0.5 where the true W1 is 1.0.
    while i < n and j < m:
        x = x1[i] if x1[i] <= x2[j] else x2[j]
        dist += abs(cdf1 - cdf2) * (x - last_x)
        last_x = x
        # Advance every tie at this position before moving on.
        while i < n and x1[i] == x:
            i += 1
            cdf1 = i / n
        while j < m and x2[j] == x:
            j += 1
            cdf2 = j / m

    while i < n:
        x = x1[i]
        dist += abs(cdf1 - cdf2) * (x - last_x)
        last_x = x
        i += 1
        cdf1 = i / n

    while j < m:
        x = x2[j]
        dist += abs(cdf1 - cdf2) * (x - last_x)
        last_x = x
        j += 1
        cdf2 = j / m

    return float(dist)

validation_scorer(estimator, X, y)

Scorer callable for cross_validate and GridSearchCV.

Follows the scorer(estimator, X, y) signature and the greater-is-better convention, so it can be passed directly as scoring=.

Source code in src/oversampleqa/estimator.py

def validation_scorer(
    estimator: OversamplingValidator,
    X: NDArray[np.floating],
    y: NDArray[np.integer],
) -> float:
    """Scorer callable for ``cross_validate`` and ``GridSearchCV``.

    Follows the ``scorer(estimator, X, y)`` signature and the greater-is-better
    convention, so it can be passed directly as ``scoring=``.
    """
    return estimator.score(X, y)

boundary_violation_rate(synthetic, X_real, y_real, minority_label, *, k=5, metric='hassanat', metric_kwargs=None)

Fraction of synthetic points sitting in majority territory.

Measures the failure this package exists to detect, per point and without a hold-out -- so it can still be reported when the minority is too small for :func:~oversampleqa.validate_oversampling's hold-out guard.

Two versions are returned because they answer different questions:

strict_rate Fraction whose all k nearest real neighbours are majority. Unambiguous violations. graded_rate Mean majority fraction among the k neighbours. Sensitive to points drifting toward the boundary before they cross it.

This is unrelated to :func:~oversampleqa.noise_sensitivity_diagnostic, which measures how the error rate responds to injected label noise -- a different question, so the two do not overlap.

Returns

BoundaryReport

Source code in src/oversampleqa/fidelity.py

def boundary_violation_rate(
    synthetic: NDArray[np.floating],
    X_real: NDArray[np.floating],
    y_real: NDArray[np.integer],
    minority_label: int,
    *,
    k: int = 5,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
) -> BoundaryReport:
    """Fraction of synthetic points sitting in majority territory.

    Measures the failure this package exists to detect, per point and without a
    hold-out -- so it can still be reported when the minority is too small for
    :func:`~oversampleqa.validate_oversampling`'s hold-out guard.

    Two versions are returned because they answer different questions:

    ``strict_rate``
        Fraction whose **all** ``k`` nearest real neighbours are majority.
        Unambiguous violations.
    ``graded_rate``
        Mean majority fraction among the ``k`` neighbours. Sensitive to points
        drifting toward the boundary before they cross it.

    This is unrelated to
    :func:`~oversampleqa.noise_sensitivity_diagnostic`, which measures how the
    error rate responds to injected *label noise* -- a different question, so
    the two do not overlap.

    Returns
    -------
    BoundaryReport
    """
    synthetic = np.asarray(synthetic, dtype=float)
    X_real = np.asarray(X_real, dtype=float)
    y_real = np.asarray(y_real)
    if len(synthetic) == 0:
        raise ValidationError("synthetic is empty; nothing to measure")
    if len(X_real) < k:
        raise ValidationError(
            f"need at least k={k} real points to inspect neighbours, got {len(X_real)}"
        )

    distances = distance_matrix(synthetic, X_real, metric, **(metric_kwargs or {}))
    neighbours = np.argsort(distances, axis=1, kind="stable")[:, :k]
    is_majority = (y_real != minority_label)[neighbours]

    return BoundaryReport(
        strict_rate=float(is_majority.all(axis=1).mean()),
        graded_rate=float(is_majority.mean()),
        k=k,
        metric=metric,
        n_synthetic=len(synthetic),
    )

fidelity_report(X, y, minority_label, oversampler, *, metric='hassanat', k=5, hidden_ratio=0.1, random_state=42, include_utility=False)

Run the full fidelity suite for one oversampler.

Parameters

X, y : ndarray Full dataset. minority_label : int Minority class label. oversampler : object An imbalanced-learn sampler. metric : str, default="hassanat" Distance metric for every geometric measure. k : int, default=5 Neighbours for the manifold and boundary estimates. hidden_ratio : float, default=0.1 Fraction held out, matching validate_oversampling. include_utility : bool, default=False Fit models to measure downstream gain. Off by default because it is far slower than the geometric measures.

Returns

FidelityReport

Source code in src/oversampleqa/fidelity.py

def fidelity_report(
    X: NDArray[np.floating],
    y: NDArray[np.integer],
    minority_label: int,
    oversampler: Any,
    *,
    metric: str = "hassanat",
    k: int = 5,
    hidden_ratio: float = 0.1,
    random_state: int | None = 42,
    include_utility: bool = False,
) -> FidelityReport:
    """Run the full fidelity suite for one oversampler.

    Parameters
    ----------
    X, y : ndarray
        Full dataset.
    minority_label : int
        Minority class label.
    oversampler : object
        An ``imbalanced-learn`` sampler.
    metric : str, default="hassanat"
        Distance metric for every geometric measure.
    k : int, default=5
        Neighbours for the manifold and boundary estimates.
    hidden_ratio : float, default=0.1
        Fraction held out, matching ``validate_oversampling``.
    include_utility : bool, default=False
        Fit models to measure downstream gain. Off by default because it is far
        slower than the geometric measures.

    Returns
    -------
    FidelityReport
    """
    from .validator import (
        extract_synthetic_samples,
        prepare_validation_split,
        validate_oversampling,
    )

    labels = np.unique(y)
    if len(labels) != 2:
        raise ValidationError("fidelity_report expects binary labels")
    majority_label = int(labels[labels != minority_label][0])

    split = prepare_validation_split(
        X, y, minority_label, majority_label, hidden_ratio, random_state=random_state
    )
    X_res, y_res = oversampler.fit_resample(split.X_train, split.y_train)
    synthetic = extract_synthetic_samples(split.X_train, X_res, y_res, minority_label)
    if len(synthetic) == 0:
        raise ValidationError(
            f"{type(oversampler).__name__} produced no synthetic samples"
        )

    error_rate = validate_oversampling(
        X,
        y,
        minority_label,
        oversampler,
        hidden_ratio=hidden_ratio,
        metric=metric,
        random_state=random_state,
    )

    # return_details=False always yields a float; narrow it at the boundary
    # rather than suppressing the union.
    if isinstance(error_rate, ValidationDetails):  # pragma: no cover
        raise ValidationError(
            "validate_oversampling(return_details=False) must return a float"
        )

    utility = None
    if include_utility:
        utility = downstream_utility(X, y, oversampler, random_state=random_state)

    return FidelityReport(
        error_rate=float(error_rate),
        manifold=precision_recall_density_coverage(
            synthetic, split.reference_minority, k=k, metric=metric
        ),
        memorisation=memorisation_report(synthetic, split.fit_minority, metric=metric),
        boundary=boundary_violation_rate(
            synthetic, X, y, minority_label, k=k, metric=metric
        ),
        utility=utility,
    )

memorisation_report(synthetic, train_minority, *, metric='hassanat', metric_kwargs=None, quantiles=(0.01, 0.05))

Assess how much of the output is copied from the training minority.

The headline is distance_ratio: the median distance from a synthetic point to its nearest training point, over the median nearest-neighbour distance within the real minority. That denominator is what makes the number legible -- it is the natural spacing of real data, so a ratio well below 1 says the generator sits closer to its training points than real points sit to each other.

Near-duplicate thresholds come from the same distribution rather than an absolute tolerance, so they mean the same thing on any dataset.

Parameters

synthetic, train_minority : ndarray Synthetic points and the minority data the sampler was fitted on. quantiles : tuple of float, default=(0.01, 0.05) Quantiles of the real nearest-neighbour distance distribution to use as near-duplicate thresholds.

Returns

MemorisationReport

Source code in src/oversampleqa/fidelity.py

def memorisation_report(
    synthetic: NDArray[np.floating],
    train_minority: NDArray[np.floating],
    *,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
    quantiles: tuple[float, ...] = (0.01, 0.05),
) -> MemorisationReport:
    """Assess how much of the output is copied from the training minority.

    The headline is ``distance_ratio``: the median distance from a synthetic
    point to its nearest training point, over the median nearest-neighbour
    distance *within* the real minority. That denominator is what makes the
    number legible -- it is the natural spacing of real data, so a ratio well
    below 1 says the generator sits closer to its training points than real
    points sit to each other.

    Near-duplicate thresholds come from the same distribution rather than an
    absolute tolerance, so they mean the same thing on any dataset.

    Parameters
    ----------
    synthetic, train_minority : ndarray
        Synthetic points and the minority data the sampler was fitted on.
    quantiles : tuple of float, default=(0.01, 0.05)
        Quantiles of the real nearest-neighbour distance distribution to use as
        near-duplicate thresholds.

    Returns
    -------
    MemorisationReport
    """
    synthetic = np.asarray(synthetic, dtype=float)
    train_minority = np.asarray(train_minority, dtype=float)
    if len(synthetic) == 0:
        raise ValidationError("synthetic is empty; nothing to measure")
    if len(train_minority) < 2:
        raise ValidationError(
            "memorisation needs at least 2 training minority points to establish "
            f"the real spacing; got {len(train_minority)}"
        )

    kwargs = metric_kwargs or {}
    to_train = distance_matrix(synthetic, train_minority, metric, **kwargs).min(axis=1)

    within = distance_matrix(train_minority, train_minority, metric, **kwargs)
    within = np.array(within, copy=True)
    np.fill_diagonal(within, np.inf)
    real_nn = within.min(axis=1)

    median_to_train = float(np.median(to_train))
    median_real_nn = float(np.median(real_nn))
    ratio = median_to_train / median_real_nn if median_real_nn > 0 else float("nan")

    near_rates = {
        q: float((to_train <= np.quantile(real_nn, q)).mean()) for q in quantiles
    }

    # Not `== 0.0`. Metrics computed through the BLAS gram trick -- euclidean
    # among them -- lose the last bits to cancellation for identical points and
    # return ~1e-8 rather than exactly zero, while direct formulas such as
    # hassanat return exact zeros. An exact test would therefore report a
    # different duplicate rate for the same data depending on the metric. The
    # tolerance is scaled by the real spacing so it stays scale-free.
    duplicate_tolerance = max(median_real_nn * 1e-6, np.finfo(float).eps * 100)

    return MemorisationReport(
        distance_ratio=ratio,
        exact_duplicate_rate=float((to_train <= duplicate_tolerance).mean()),
        near_duplicate_rates=near_rates,
        median_distance_to_train=median_to_train,
        median_real_nn_distance=median_real_nn,
        metric=metric,
        n_synthetic=len(synthetic),
    )

precision_recall_density_coverage(synthetic, real, *, k=5, metric='hassanat', metric_kwargs=None)

Estimate fidelity and diversity from k-NN manifolds.

The real manifold is the union of hyperspheres centred on each real point with radius its k-th nearest neighbour distance; the synthetic manifold is the same construction on synthetic points.

Parameters

synthetic, real : ndarray Synthetic points and real held-out minority points. k : int, default=5 Neighbours defining each sphere. These metrics are sensitive to k; use :func:sweep_k rather than trusting one value. metric : str, default="hassanat" Any metric from the package registry.

Returns

ManifoldMetrics

Raises

ValidationError If synthetic is empty or real has fewer than k + 1 points.

Source code in src/oversampleqa/fidelity.py

def precision_recall_density_coverage(
    synthetic: NDArray[np.floating],
    real: NDArray[np.floating],
    *,
    k: int = 5,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
) -> ManifoldMetrics:
    """Estimate fidelity and diversity from k-NN manifolds.

    The real manifold is the union of hyperspheres centred on each real point
    with radius its k-th nearest neighbour distance; the synthetic manifold is
    the same construction on synthetic points.

    Parameters
    ----------
    synthetic, real : ndarray
        Synthetic points and real held-out minority points.
    k : int, default=5
        Neighbours defining each sphere. **These metrics are sensitive to k**;
        use :func:`sweep_k` rather than trusting one value.
    metric : str, default="hassanat"
        Any metric from the package registry.

    Returns
    -------
    ManifoldMetrics

    Raises
    ------
    ValidationError
        If ``synthetic`` is empty or ``real`` has fewer than ``k + 1`` points.
    """
    synthetic = np.asarray(synthetic, dtype=float)
    real = np.asarray(real, dtype=float)
    _check_sizes(synthetic, real, k)
    _warn_if_high_dimensional(real)

    kwargs = metric_kwargs or {}
    real_radii = _knn_radii(real, k, metric, kwargs)
    cross = distance_matrix(synthetic, real, metric, **kwargs)

    # Precision: a synthetic point is inside the real manifold if it falls in
    # any real point's sphere.
    inside_real = cross <= real_radii[None, :]
    precision = float(inside_real.any(axis=1).mean())

    # Density: how many real spheres contain it, normalised by k. Counting
    # rather than thresholding is what stops one outsized outlier sphere from
    # certifying every synthetic point at once.
    density = float(inside_real.sum(axis=1).mean() / k)

    # Coverage: fraction of real points with a synthetic point in their sphere.
    coverage = float(inside_real.any(axis=0).mean())

    # Recall needs the synthetic manifold, so it needs enough synthetic points.
    if len(synthetic) >= k + 1:
        synthetic_radii = _knn_radii(synthetic, k, metric, kwargs)
        inside_synthetic = synthetic_radii[None, :] >= cross.T
        recall = float(inside_synthetic.any(axis=1).mean())
    else:
        recall = float("nan")

    return ManifoldMetrics(
        precision=precision,
        recall=recall,
        density=density,
        coverage=coverage,
        k=k,
        metric=metric,
        n_synthetic=len(synthetic),
        n_real=len(real),
    )

cross_match_test(synthetic, real, *, metric='hassanat', metric_kwargs=None, n_permutations=999, parents=None, n_subsamples=9, random_state=42)

Rosenbaum cross-match test, with a greedy matching.

Pair up the pooled sample and count how many pairs join the two samples. Well-mixed samples yield many cross pairs, so the p-value is left-tailed.

.. note::

Rosenbaum's test uses optimal non-bipartite matching, which minimises total matched distance and admits an exact null distribution. This implementation uses a greedy nearest-available matching instead, so the exact distribution does not apply and the p-value comes from permutation. The greedy statistic is generally close but not identical; treat it as an approximation to the published test rather than the test itself.

Returns

TwoSampleTestResult

Source code in src/oversampleqa/inference.py

def cross_match_test(
    synthetic: NDArray[np.floating],
    real: NDArray[np.floating],
    *,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
    n_permutations: int = 999,
    parents: NDArray[np.integer] | None = None,
    n_subsamples: int = 9,
    random_state: RandomStateLike = 42,
) -> TwoSampleTestResult:
    """Rosenbaum cross-match test, with a greedy matching.

    Pair up the pooled sample and count how many pairs join the two samples.
    Well-mixed samples yield many cross pairs, so the p-value is left-tailed.

    .. note::

       Rosenbaum's test uses **optimal** non-bipartite matching, which
       minimises total matched distance and admits an exact null distribution.
       This implementation uses a greedy nearest-available matching instead, so
       the exact distribution does not apply and the p-value comes from
       permutation. The greedy statistic is generally close but not identical;
       treat it as an approximation to the published test rather than the test
       itself.

    Returns
    -------
    TwoSampleTestResult
    """
    n1, n2 = len(synthetic), len(real)
    if n1 == 0 or n2 == 0:
        raise ValidationError("both samples must be non-empty")

    if parents is not None:
        # Points sharing a parent are not exchangeable, so permuting
        # them individually gives a null that is too tight. Subsample
        # one per parent and combine; see _combine_blocked.
        inner = as_generator(random_state)
        return _combine_blocked(
            lambda subsample: cross_match_test(
                subsample,
                real,
                metric=metric,
                metric_kwargs=metric_kwargs,
                n_permutations=n_permutations,
                random_state=inner,
            ),
            synthetic,
            parents,
            n_subsamples,
            inner,
        )

    distances = _pooled_distances(synthetic, real, metric, metric_kwargs)
    labels = np.concatenate([np.zeros(n1, dtype=int), np.ones(n2, dtype=int)])

    observed = _greedy_cross_matches(distances, labels)

    rng = as_generator(random_state)
    null = [
        _greedy_cross_matches(distances, rng.permutation(labels))
        for _ in range(n_permutations)
    ]
    null_arr = np.asarray(null)
    p_perm = float((np.sum(null_arr <= observed) + 1) / (n_permutations + 1))

    return TwoSampleTestResult(
        name="rosenbaum_cross_match_greedy",
        statistic=float(observed),
        p_value=p_perm,
        n_synthetic=n1,
        n_real=n2,
        n_permutations=n_permutations,
        null_statistics=tuple(float(v) for v in null_arr),
    )

mst_two_sample_test(synthetic, real, *, metric='hassanat', metric_kwargs=None, n_permutations=999, parents=None, n_subsamples=9, random_state=42)

Friedman-Rafsky minimum-spanning-tree two-sample test.

Build the MST on the pooled sample and count edges joining the two samples. Well-mixed samples produce many cross edges; separated ones produce few, so small counts are evidence against equality and the p-value is left-tailed.

The same power caveat as :func:nn_two_sample_test applies.

Returns

TwoSampleTestResult

Source code in src/oversampleqa/inference.py

def mst_two_sample_test(
    synthetic: NDArray[np.floating],
    real: NDArray[np.floating],
    *,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
    n_permutations: int = 999,
    parents: NDArray[np.integer] | None = None,
    n_subsamples: int = 9,
    random_state: RandomStateLike = 42,
) -> TwoSampleTestResult:
    """Friedman-Rafsky minimum-spanning-tree two-sample test.

    Build the MST on the pooled sample and count edges joining the two samples.
    Well-mixed samples produce many cross edges; separated ones produce few, so
    **small** counts are evidence against equality and the p-value is
    left-tailed.

    The same power caveat as :func:`nn_two_sample_test` applies.

    Returns
    -------
    TwoSampleTestResult
    """
    n1, n2 = len(synthetic), len(real)
    if n1 == 0 or n2 == 0:
        raise ValidationError("both samples must be non-empty")

    if parents is not None:
        # Points sharing a parent are not exchangeable, so permuting
        # them individually gives a null that is too tight. Subsample
        # one per parent and combine; see _combine_blocked.
        inner = as_generator(random_state)
        return _combine_blocked(
            lambda subsample: mst_two_sample_test(
                subsample,
                real,
                metric=metric,
                metric_kwargs=metric_kwargs,
                n_permutations=n_permutations,
                random_state=inner,
            ),
            synthetic,
            parents,
            n_subsamples,
            inner,
        )

    distances = _pooled_distances(synthetic, real, metric, metric_kwargs)
    labels = np.concatenate([np.zeros(n1, dtype=int), np.ones(n2, dtype=int)])

    observed = _mst_cross_edges(distances, labels)

    rng = as_generator(random_state)
    null = [
        _mst_cross_edges(distances, rng.permutation(labels))
        for _ in range(n_permutations)
    ]
    null_arr = np.asarray(null)
    # Left-tailed: few cross edges means the samples separate.
    p_perm = float((np.sum(null_arr <= observed) + 1) / (n_permutations + 1))

    return TwoSampleTestResult(
        name="friedman_rafsky_mst",
        statistic=float(observed),
        p_value=p_perm,
        n_synthetic=n1,
        n_real=n2,
        n_permutations=n_permutations,
        null_statistics=tuple(float(v) for v in null_arr),
    )

nn_two_sample_test(synthetic, real, *, k=3, metric='hassanat', metric_kwargs=None, n_permutations=999, parents=None, n_subsamples=9, random_state=42)

Schilling-Henze nearest-neighbour two-sample test.

Of the k nearest neighbours of each point in the pooled sample, count how many share its sample label. If the two samples come from the same distribution, neighbours are labelled roughly at the base rate; if they are separated, points cluster with their own kind and the count rises.

Applied to synthetic points against held-out real minority points, this tests the question a user actually has: are these synthetic points distributionally indistinguishable from real ones? A high p-value is evidence of good synthesis.

.. warning::

Failing to reject is not proof of equality. The power of every nearest-neighbour test collapses as dimension grows, so on high-dimensional data a large p-value may reflect a lack of power rather than genuine similarity. Always read it next to n_synthetic and n_real, which are returned for exactly this reason.

Parameters

synthetic, real : ndarray The two samples. k : int, default=3 Neighbours considered per point. metric : str, default="hassanat" Any metric from the package registry, so hassanat composes with the inferential layer. n_permutations : int, default=999 Permutations behind the p-value. The pooled distance matrix is computed once and reused; permutations only relabel. random_state : int, Generator, SeedSequence or None, default=42 Seeds the permutations.

Returns

TwoSampleTestResult Carries both the permutation p-value and the asymptotic normal approximation, so the user can see where they disagree.

Source code in src/oversampleqa/inference.py

def nn_two_sample_test(
    synthetic: NDArray[np.floating],
    real: NDArray[np.floating],
    *,
    k: int = 3,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
    n_permutations: int = 999,
    parents: NDArray[np.integer] | None = None,
    n_subsamples: int = 9,
    random_state: RandomStateLike = 42,
) -> TwoSampleTestResult:
    """Schilling-Henze nearest-neighbour two-sample test.

    Of the ``k`` nearest neighbours of each point in the pooled sample, count
    how many share its sample label. If the two samples come from the same
    distribution, neighbours are labelled roughly at the base rate; if they are
    separated, points cluster with their own kind and the count rises.

    Applied to synthetic points against held-out real minority points, this
    tests the question a user actually has: *are these synthetic points
    distributionally indistinguishable from real ones?* A **high p-value is
    evidence of good synthesis**.

    .. warning::

       **Failing to reject is not proof of equality.** The power of every
       nearest-neighbour test collapses as dimension grows, so on
       high-dimensional data a large p-value may reflect a lack of power rather
       than genuine similarity. Always read it next to ``n_synthetic`` and
       ``n_real``, which are returned for exactly this reason.

    Parameters
    ----------
    synthetic, real : ndarray
        The two samples.
    k : int, default=3
        Neighbours considered per point.
    metric : str, default="hassanat"
        Any metric from the package registry, so ``hassanat`` composes with the
        inferential layer.
    n_permutations : int, default=999
        Permutations behind the p-value. The pooled distance matrix is computed
        once and reused; permutations only relabel.
    random_state : int, Generator, SeedSequence or None, default=42
        Seeds the permutations.

    Returns
    -------
    TwoSampleTestResult
        Carries both the permutation p-value and the asymptotic normal
        approximation, so the user can see where they disagree.
    """
    n1, n2 = len(synthetic), len(real)
    if n1 == 0 or n2 == 0:
        raise ValidationError("both samples must be non-empty")
    n = n1 + n2
    if k >= n:
        raise ValueError(f"k={k} must be smaller than the pooled size {n}")

    if parents is not None:
        # Points sharing a parent are not exchangeable, so permuting
        # them individually gives a null that is too tight. Subsample
        # one per parent and combine; see _combine_blocked.
        inner = as_generator(random_state)
        return _combine_blocked(
            lambda subsample: nn_two_sample_test(
                subsample,
                real,
                k=k,
                metric=metric,
                metric_kwargs=metric_kwargs,
                n_permutations=n_permutations,
                random_state=inner,
            ),
            synthetic,
            parents,
            n_subsamples,
            inner,
        )

    distances = _pooled_distances(synthetic, real, metric, metric_kwargs)
    labels = np.concatenate([np.zeros(n1, dtype=int), np.ones(n2, dtype=int)])

    observed = _nn_coincidences(distances, labels, k)

    rng = as_generator(random_state)
    null: list[int] = []
    for _ in range(n_permutations):
        null.append(_nn_coincidences(distances, rng.permutation(labels), k))

    null_arr = np.asarray(null)
    # +1 in both terms: the observed value is itself one draw from the null,
    # which keeps the p-value valid (never exactly zero).
    p_perm = float((np.sum(null_arr >= observed) + 1) / (n_permutations + 1))

    # Asymptotic normal approximation (Schilling 1986).
    lam1, lam2 = n1 / n, n2 / n
    mean = n * k * (lam1**2 + lam2**2)
    var = n * k * (lam1 * lam2 + 4 * lam1**2 * lam2**2)
    p_asym = (
        float(1.0 - stats.norm.cdf((observed - mean) / np.sqrt(var)))
        if var > 0
        else float("nan")
    )

    return TwoSampleTestResult(
        name="schilling_henze_nn",
        statistic=float(observed),
        p_value=p_perm,
        asymptotic_p_value=p_asym,
        n_synthetic=n1,
        n_real=n2,
        n_permutations=n_permutations,
        null_statistics=tuple(float(v) for v in null_arr),
    )

null_error_rate(X, y, minority_label, observed, *, hidden_ratio=0.1, metric='hassanat', metric_kwargs=None, n_draws=200, min_hidden=5, random_state=42)

Calibrate an observed error rate against ideal and worst-case references.

The null is built by scoring real held-out minority points through the identical pipeline. Those points are, by construction, drawn from the true minority distribution, so their error rate is what a perfect generator would score. Anything an actual oversampler achieves can then be read as a position relative to that.

The ceiling uses points drawn from the majority region -- what a deliberately bad generator produces -- bounding the other end of the scale.

Parameters

X, y : ndarray Input data and labels. minority_label : int Label of the minority class. observed : float The error rate to interpret, e.g. from :func:~oversampleqa.validate_oversampling. hidden_ratio : float, default=0.1 Fraction held out. Must match the run that produced observed, or the comparison is meaningless. metric : str, default="hassanat" Distance metric. Must also match. n_draws : int, default=200 Independent splits behind the null distribution. min_hidden : int, default=5 Minimum held-out minority points per draw. random_state : int, Generator, SeedSequence or None, default=42 Seeds the draws.

Returns

NullCalibration

Raises

ValidationError If the labels are not binary or the minority is too small.

Notes

hidden_ratio and metric must match the run that produced observed. The error rate's scale depends on both, so calibrating against a null computed with different settings compares two different quantities.

Source code in src/oversampleqa/inference.py

def null_error_rate(
    X: NDArray[np.floating],
    y: NDArray[np.integer],
    minority_label: int,
    observed: float,
    *,
    hidden_ratio: float = 0.1,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
    n_draws: int = 200,
    min_hidden: int = 5,
    random_state: RandomStateLike = 42,
) -> NullCalibration:
    """Calibrate an observed error rate against ideal and worst-case references.

    The null is built by scoring **real held-out minority points** through the
    identical pipeline. Those points are, by construction, drawn from the true
    minority distribution, so their error rate is what a perfect generator would
    score. Anything an actual oversampler achieves can then be read as a
    position relative to that.

    The ceiling uses points drawn from the majority region -- what a
    deliberately bad generator produces -- bounding the other end of the scale.

    Parameters
    ----------
    X, y : ndarray
        Input data and labels.
    minority_label : int
        Label of the minority class.
    observed : float
        The error rate to interpret, e.g. from
        :func:`~oversampleqa.validate_oversampling`.
    hidden_ratio : float, default=0.1
        Fraction held out. Must match the run that produced ``observed``, or
        the comparison is meaningless.
    metric : str, default="hassanat"
        Distance metric. Must also match.
    n_draws : int, default=200
        Independent splits behind the null distribution.
    min_hidden : int, default=5
        Minimum held-out minority points per draw.
    random_state : int, Generator, SeedSequence or None, default=42
        Seeds the draws.

    Returns
    -------
    NullCalibration

    Raises
    ------
    ValidationError
        If the labels are not binary or the minority is too small.

    Notes
    -----
    ``hidden_ratio`` and ``metric`` must match the run that produced
    ``observed``. The error rate's scale depends on both, so calibrating
    against a null computed with different settings compares two different
    quantities.
    """
    require_pointwise_metric(metric)
    labels = np.unique(y)
    if len(labels) != 2:
        raise ValidationError(
            f"null_error_rate expects binary labels; got {len(labels)} distinct values"
        )
    if minority_label not in labels:
        raise ValidationError(f"minority_label {minority_label} not found in y")
    majority_label = int(labels[labels != minority_label][0])

    if n_draws < 1:
        raise ValueError(f"n_draws must be at least 1; got {n_draws}")

    generators = spawn_generators(random_state, n_draws)
    majority = X[y != minority_label]
    n_minority = int(np.sum(y == minority_label))

    # Three disjoint minority pieces are needed, not two. See the estimand note
    # in the docstring: the null candidates must be scored against the same
    # reference the observed synthetic points were, and cannot be part of it.
    ratio = hidden_ratio
    if int(n_minority * ratio) < min_hidden or n_minority - 2 * int(
        n_minority * ratio
    ) < 1:
        raise ValidationError(
            f"A minority class of {n_minority} cannot support the calibration "
            f"split at hidden_ratio={ratio:.3g} and min_hidden={min_hidden}. "
            "Calibration needs three disjoint minority sets -- one standing in "
            "for the sampler's training data, one common reference, and one "
            "supplying the real null candidates -- because scoring the null "
            "against a reference it belongs to measures nothing. Supply more "
            "minority data or lower min_hidden."
        )

    null_rates: list[float] = []
    ceiling_rates: list[float] = []

    for gen in generators:
        split = prepare_validation_split(
            X,
            y,
            minority_label,
            majority_label,
            hidden_ratio,
            reference="hidden_minority",
            min_hidden=min_hidden,
            random_state=gen,
        )
        # The common minority reference. `validate_oversampling` scores its
        # synthetic points against exactly this set, so the null must too --
        # previously the null used `fit_minority` instead, which is a different
        # and much larger set, so the two rates were not the same quantity and
        # the calibration compared observed against a null of something else.
        reference = split.reference_minority

        # Null candidates: real minority points held out from both the
        # reference and the notional training set. They stand in for synthetic
        # points, so they must be disjoint from the reference they are scored
        # against; carving them from `fit_minority` guarantees that.
        n_null = min(len(reference), max(len(split.fit_minority) - 1, 0))
        if n_null == 0:
            null_rates.append(float("nan"))
            ceiling_rates.append(float("nan"))
            continue
        null_idx = gen.choice(len(split.fit_minority), size=n_null, replace=False)
        null_candidates = split.fit_minority[null_idx]

        null_rates.append(
            _score_against(
                null_candidates,
                split.hid_majority,
                reference,
                metric,
                metric_kwargs,
            )
        )

        # The ceiling: majority points standing in for synthetic ones, i.e. a
        # generator that has learned the wrong distribution entirely.
        #
        # Drawn from the *visible* majority. Drawing from the full majority put
        # hidden-majority points into the candidate set -- measured at 8.8% of
        # candidates, with 64% of draws affected -- and a candidate that is
        # itself in the reference sits at distance zero from it, so it is
        # counted as an error by construction. That inflated the ceiling and,
        # with it, every `scaled` position measured against it.
        visible_mask = np.ones(len(majority), dtype=bool)
        visible_mask[split.hidden_majority_index] = False
        visible_majority = majority[visible_mask]
        if len(visible_majority) == 0:
            ceiling_rates.append(float("nan"))
            continue
        n_bad = min(len(reference), len(visible_majority))
        bad_idx = gen.choice(len(visible_majority), size=n_bad, replace=False)
        ceiling_rates.append(
            _score_against(
                visible_majority[bad_idx],
                split.hid_majority,
                reference,
                metric,
                metric_kwargs,
            )
        )

    null_arr = np.asarray(null_rates, dtype=float)
    finite_null = null_arr[np.isfinite(null_arr)]
    ceiling_arr = np.asarray(ceiling_rates, dtype=float)
    finite_ceiling = ceiling_arr[np.isfinite(ceiling_arr)]

    null_mean = float(np.mean(finite_null)) if finite_null.size else float("nan")
    null_sd = (
        float(np.std(finite_null, ddof=1)) if finite_null.size > 1 else float("nan")
    )
    ceiling_mean = (
        float(np.mean(finite_ceiling)) if finite_ceiling.size else float("nan")
    )

    z = (observed - null_mean) / null_sd if null_sd and null_sd > 0 else float("nan")
    percentile = (
        float((finite_null <= observed).mean() * 100.0)
        if finite_null.size
        else float("nan")
    )
    span = ceiling_mean - null_mean
    scaled = (
        (observed - null_mean) / span if span and abs(span) > 1e-12 else float("nan")
    )

    return NullCalibration(
        observed=observed,
        null_rates=tuple(null_rates),
        ceiling_rates=tuple(ceiling_rates),
        z_score=z,
        percentile=percentile,
        scaled=scaled,
        metric=metric,
        n_draws=n_draws,
    )

calculate_error_rate(errors, total)

Return error rate given the number of errors and total samples.

Parameters:

Name	Type	Description	Default
errors	int	Number of error samples.	required
total	int	Total number of samples.	required

Returns:

Type	Description
float	Error rate in the range [0, 1], or nan when total is zero.

Notes

A zero denominator means nothing was measured. Returning 0.0 in that case would be indistinguishable from a perfect score, so nan is returned instead. Callers that aggregate error rates must use nan-aware reductions (np.nanmean) deliberately.

Source code in src/oversampleqa/metrics.py

def calculate_error_rate(errors: int, total: int) -> float:
    """Return error rate given the number of errors and total samples.

    Args:
        errors: Number of error samples.
        total: Total number of samples.

    Returns:
        Error rate in the range [0, 1], or ``nan`` when ``total`` is zero.

    Notes:
        A zero denominator means nothing was measured. Returning ``0.0`` in
        that case would be indistinguishable from a perfect score, so ``nan``
        is returned instead. Callers that aggregate error rates must use
        ``nan``-aware reductions (``np.nanmean``) deliberately.
    """
    if total == 0:
        return float("nan")
    return errors / total

check_model_fairness(y_true, y_pred, protected_attr, minority_label)

Return absolute difference in minority recall across protected groups.

Parameters:

Name	Type	Description	Default
y_true	NDArray[Any]	True labels.	required
y_pred	NDArray[Any]	Predicted labels.	required
protected_attr	NDArray[Any]	Protected group labels.	required
minority_label	int	Minority class label.	required

Returns:

Type	Description
float	Absolute recall gap between the two groups.

Source code in src/oversampleqa/metrics.py

def check_model_fairness(
    y_true: NDArray[Any],
    y_pred: NDArray[Any],
    protected_attr: NDArray[Any],
    minority_label: int,
) -> float:
    """Return absolute difference in minority recall across protected groups.

    Args:
        y_true: True labels.
        y_pred: Predicted labels.
        protected_attr: Protected group labels.
        minority_label: Minority class label.

    Returns:
        Absolute recall gap between the two groups.
    """

    from sklearn.metrics import recall_score

    groups = np.unique(protected_attr)
    if len(groups) != 2:
        raise ValueError("protected_attr must have exactly two groups")

    recalls = []
    for g in groups:
        mask = protected_attr == g
        if mask.sum() == 0:
            recalls.append(0.0)
        else:
            recalls.append(
                recall_score(
                    y_true[mask] == minority_label, y_pred[mask] == minority_label
                )
            )

    return abs(recalls[0] - recalls[1])

confidence_ratio(dist_min, dist_maj)

Return ratio between distances to minority and majority classes.

Parameters:

Name	Type	Description	Default
dist_min	float	Distance to minority class.	required
dist_maj	float	Distance to majority class.	required

Returns:

Type	Description
float	Ratio dist_min / dist_maj (inf if dist_maj is zero).

Source code in src/oversampleqa/metrics.py

def confidence_ratio(dist_min: float, dist_maj: float) -> float:
    """Return ratio between distances to minority and majority classes.

    Args:
        dist_min: Distance to minority class.
        dist_maj: Distance to majority class.

    Returns:
        Ratio ``dist_min / dist_maj`` (inf if ``dist_maj`` is zero).
    """
    if dist_maj == 0:
        return float("inf")
    return dist_min / dist_maj

duplication_rate(synthetic, reference, *, atol=0.0)

Fraction of synthetic points that coincide with a reference point.

Parameters

synthetic : ndarray Synthetic samples of shape (n_synthetic, n_features). reference : ndarray Real samples the synthetic points may have been copied from. atol : float, default=0.0 Absolute tolerance for treating a synthetic point as a duplicate. The default of 0.0 requires exact equality.

Returns

float Value in [0, 1]; nan when there are no synthetic samples.

Notes

An oversampler that duplicates rather than synthesises -- such as RandomOverSampler -- scores 1.0. Its validation error rate is then uninformative about synthesis quality, because every "synthetic" point sits exactly on top of a real one.

Source code in src/oversampleqa/metrics.py

def duplication_rate(
    synthetic: NDArray[np.floating],
    reference: NDArray[np.floating],
    *,
    atol: float = 0.0,
) -> float:
    """Fraction of synthetic points that coincide with a reference point.

    Parameters
    ----------
    synthetic : ndarray
        Synthetic samples of shape ``(n_synthetic, n_features)``.
    reference : ndarray
        Real samples the synthetic points may have been copied from.
    atol : float, default=0.0
        Absolute tolerance for treating a synthetic point as a duplicate.
        The default of ``0.0`` requires exact equality.

    Returns
    -------
    float
        Value in ``[0, 1]``; ``nan`` when there are no synthetic samples.

    Notes
    -----
    An oversampler that duplicates rather than synthesises -- such as
    ``RandomOverSampler`` -- scores ``1.0``. Its validation error rate is then
    uninformative about synthesis quality, because every "synthetic" point sits
    exactly on top of a real one.
    """
    if len(synthetic) == 0:
        return float("nan")
    if len(reference) == 0:
        return 0.0

    matches = 0
    for point in synthetic:
        deltas = np.abs(reference - point).max(axis=1)
        if bool(np.any(deltas <= atol)):
            matches += 1
    return matches / len(synthetic)

local_density_divergence(synthetic_samples, reference_samples, k=5)

Compute divergence of local densities between synthetic and reference data.

This metric compares the average distance to the k nearest neighbours for synthetic samples against the same statistic computed on the reference samples themselves. A higher value indicates that synthetic samples reside in sparser regions of the space compared to the reference distribution.

Parameters

synthetic_samples, reference_samples : ndarray Arrays of shape (n_samples, n_features) representing synthetic and reference data respectively. k : int, default=5 Number of nearest neighbours to consider when estimating local density.

Returns

float Relative difference in mean neighbourhood radii. 0.0 indicates that both sets have similar local density.

Source code in src/oversampleqa/metrics.py

def local_density_divergence(
    synthetic_samples: NDArray[np.floating],
    reference_samples: NDArray[np.floating],
    k: int = 5,
) -> float:
    """Compute divergence of local densities between synthetic and reference data.

    This metric compares the average distance to the ``k`` nearest neighbours
    for synthetic samples against the same statistic computed on the reference
    samples themselves. A higher value indicates that synthetic samples reside
    in sparser regions of the space compared to the reference distribution.

    Parameters
    ----------
    synthetic_samples, reference_samples : ndarray
        Arrays of shape ``(n_samples, n_features)`` representing synthetic and
        reference data respectively.
    k : int, default=5
        Number of nearest neighbours to consider when estimating local density.

    Returns
    -------
    float
        Relative difference in mean neighbourhood radii. ``0.0`` indicates that
        both sets have similar local density.
    """

    if np.array_equal(synthetic_samples, reference_samples):
        return 0.0

    if len(reference_samples) < 2 or len(synthetic_samples) == 0:
        return 0.0

    from sklearn.neighbors import NearestNeighbors

    k = min(k, len(reference_samples) - 1)

    nbrs_ref = NearestNeighbors(n_neighbors=k + 1).fit(reference_samples)
    ref_dists, _ = nbrs_ref.kneighbors(reference_samples)
    mean_ref = ref_dists[:, 1:].mean()

    nbrs_syn = NearestNeighbors(n_neighbors=k).fit(reference_samples)
    syn_dists, _ = nbrs_syn.kneighbors(synthetic_samples)
    mean_syn = syn_dists.mean()

    if mean_ref == 0:
        return 0.0
    divergence: float = (mean_syn - mean_ref) / mean_ref
    return divergence

minority_recall_loss(y_true, y_pred, minority_label)

Return recall loss for the minority class.

Parameters

y_true, y_pred : ndarray True and predicted class labels. minority_label : int Label of the minority class.

Returns

float 1 - recall for the minority class.

Source code in src/oversampleqa/metrics.py

def minority_recall_loss(
    y_true: NDArray[Any], y_pred: NDArray[Any], minority_label: int
) -> float:
    """Return recall loss for the minority class.

    Parameters
    ----------
    y_true, y_pred : ndarray
        True and predicted class labels.
    minority_label : int
        Label of the minority class.

    Returns
    -------
    float
        ``1 - recall`` for the minority class.
    """

    from sklearn.metrics import recall_score

    recall = recall_score(y_true == minority_label, y_pred == minority_label)
    achieved: float = recall
    return 1.0 - achieved

noise_sensitivity_diagnostic(X, y, minority_label, oversampler, noise_levels=None, hidden_ratio=0.1, metric='hassanat', random_state=None)

Evaluate error rate under different label noise levels.

Parameters:

Name	Type	Description	Default
X	NDArray[floating]	Feature matrix.	required
y	NDArray[Any]	Target labels.	required
minority_label	int	Minority class label.	required
oversampler	Any	Oversampler instance.	required
noise_levels	list[float] | None	Noise levels to evaluate.	None
hidden_ratio	float	Fraction of majority to hide.	0.1
metric	str	Distance metric name.	'hassanat'
random_state	int | None	Optional random seed.	None

Returns:

Type	Description
DataFrame	DataFrame with noise, error_rate and n_flipped -- the number
DataFrame	of labels actually changed, so the applied noise can be checked against
DataFrame	the requested level rather than assumed.

Raises:

Type	Description
ValueError	If y contains fewer than two classes, leaving no label to flip to.

Notes

Replacement labels are drawn from the other classes. Drawing from all classes, as this used to, lets a selected point keep its own label, so the realised noise was noise * (k - 1) / k: on binary data -- this package's main case -- half the requested level. A run labelled noise=0.3 applied about 0.15, and the x-axis of every noise-sensitivity plot was overstated by that factor.

Source code in src/oversampleqa/metrics.py

def noise_sensitivity_diagnostic(
    X: NDArray[np.floating],
    y: NDArray[Any],
    minority_label: int,
    oversampler: Any,
    noise_levels: list[float] | None = None,
    hidden_ratio: float = 0.1,
    metric: str = "hassanat",
    random_state: int | None = None,
) -> pd.DataFrame:
    """Evaluate error rate under different label noise levels.

    Args:
        X: Feature matrix.
        y: Target labels.
        minority_label: Minority class label.
        oversampler: Oversampler instance.
        noise_levels: Noise levels to evaluate.
        hidden_ratio: Fraction of majority to hide.
        metric: Distance metric name.
        random_state: Optional random seed.

    Returns:
        DataFrame with ``noise``, ``error_rate`` and ``n_flipped`` -- the number
        of labels actually changed, so the applied noise can be checked against
        the requested level rather than assumed.

    Raises:
        ValueError: If ``y`` contains fewer than two classes, leaving no label
            to flip to.

    Notes:
        Replacement labels are drawn from the *other* classes. Drawing from all
        classes, as this used to, lets a selected point keep its own label, so
        the realised noise was ``noise * (k - 1) / k``: on binary data -- this
        package's main case -- **half** the requested level. A run labelled
        ``noise=0.3`` applied about 0.15, and the x-axis of every
        noise-sensitivity plot was overstated by that factor.
    """

    from .validator import validate_oversampling

    noise_levels = noise_levels or [0.0, 0.1, 0.2, 0.3]
    rng = np.random.default_rng(random_state)
    results = []
    labels = np.unique(y)
    if len(labels) < 2:
        raise ValueError(
            "noise_sensitivity_diagnostic needs at least two classes: with one "
            "class there is no other label to flip to, so no noise level is "
            "distinguishable from zero."
        )

    for noise in noise_levels:
        y_noisy = y.copy()
        n_flipped = 0
        if noise > 0:
            n_flip = int(len(y) * noise)
            idx = rng.choice(len(y), n_flip, replace=False)
            y_noisy = flip_labels(y, idx, labels, rng)
            n_flipped = int(np.sum(y_noisy != y))

        err = validate_oversampling(
            X,
            y_noisy,
            minority_label=minority_label,
            oversampler=oversampler,
            hidden_ratio=hidden_ratio,
            metric=metric,
        )
        results.append(
            {"noise": noise, "error_rate": err, "n_flipped": n_flipped}
        )

    return pd.DataFrame(results)

umap_manifold_distance(real, synthetic, n_neighbors=15, random_state=None)

Return Wasserstein distance between real and synthetic data in UMAP space.

Parameters:

Name	Type	Description	Default
real	NDArray[floating]	Real samples.	required
synthetic	NDArray[floating]	Synthetic samples.	required
n_neighbors	int	UMAP neighborhood size.	15
random_state	int | None	Optional random seed.	None

Returns:

Type	Description
float	Mean Wasserstein distance across UMAP dimensions.

Source code in src/oversampleqa/metrics.py

def umap_manifold_distance(
    real: NDArray[np.floating],
    synthetic: NDArray[np.floating],
    n_neighbors: int = 15,
    random_state: int | None = None,
) -> float:
    """Return Wasserstein distance between real and synthetic data in UMAP space.

    Args:
        real: Real samples.
        synthetic: Synthetic samples.
        n_neighbors: UMAP neighborhood size.
        random_state: Optional random seed.

    Returns:
        Mean Wasserstein distance across UMAP dimensions.
    """

    from umap import UMAP

    from .extended_distances import wasserstein_1d_distance

    if len(synthetic) == 0 or len(real) == 0:
        return 0.0

    reducer = UMAP(
        n_neighbors=n_neighbors,
        n_components=2,
        random_state=random_state,
        n_jobs=1,
    )
    X = np.vstack([real, synthetic])
    embed = reducer.fit_transform(X)
    real_emb = embed[: len(real)]
    synth_emb = embed[len(real) :]
    d1 = wasserstein_1d_distance(real_emb[:, 0], synth_emb[:, 0])
    d2 = wasserstein_1d_distance(real_emb[:, 1], synth_emb[:, 1])
    return float((d1 + d2) / 2)

plot_class_balance(labels_before, labels_after, save_path=None)

Bar chart comparing class counts before and after oversampling.

Parameters

labels_before, labels_after : ndarray Class labels prior to oversampling and after applying an oversampler. save_path : str, optional If given, path to save the resulting plot. Otherwise the figure is closed and not displayed.

Source code in src/oversampleqa/plotting.py

def plot_class_balance(
    labels_before: NDArray[np.integer],
    labels_after: NDArray[np.integer],
    save_path: str | None = None,
) -> None:
    """Bar chart comparing class counts before and after oversampling.

    Parameters
    ----------
    labels_before, labels_after : ndarray
        Class labels prior to oversampling and after applying an oversampler.
    save_path : str, optional
        If given, path to save the resulting plot. Otherwise the figure is
        closed and not displayed.
    """

    counts_before = pd.Series(labels_before).value_counts().sort_index()
    counts_after = pd.Series(labels_after).value_counts().sort_index()
    df = pd.DataFrame({"before": counts_before, "after": counts_after})
    df.plot(kind="bar")
    plt.ylabel("Count")
    plt.tight_layout()
    if save_path:
        plt.savefig(save_path)
    else:
        plt.close()

plot_distance_histogram(dist_hidden, dist_minority, save_path=None)

Histogram of nearest distances to hidden majority and real minority samples.

Parameters

dist_hidden, dist_minority : ndarray Distance matrices where rows correspond to synthetic samples and columns to hidden majority or real minority samples respectively. save_path : str, optional If given, path to save the resulting plot. Otherwise the figure is closed and not displayed.

Source code in src/oversampleqa/plotting.py

def plot_distance_histogram(
    dist_hidden: NDArray[np.floating],
    dist_minority: NDArray[np.floating],
    save_path: str | None = None,
) -> None:
    """Histogram of nearest distances to hidden majority and real minority samples.

    Parameters
    ----------
    dist_hidden, dist_minority : ndarray
        Distance matrices where rows correspond to synthetic samples and
        columns to hidden majority or real minority samples respectively.
    save_path : str, optional
        If given, path to save the resulting plot. Otherwise the figure is
        closed and not displayed.
    """

    hidden_nearest = dist_hidden.min(axis=1) if dist_hidden.size else np.array([])
    minority_nearest = dist_minority.min(axis=1) if dist_minority.size else np.array([])

    plt.figure()
    if hidden_nearest.size:
        sns.histplot(hidden_nearest, color="red", alpha=0.5, label="hidden")
    if minority_nearest.size:
        sns.histplot(minority_nearest, color="blue", alpha=0.5, label="minority")
    plt.xlabel("Distance")
    plt.ylabel("Count")
    plt.legend()
    plt.tight_layout()
    if save_path:
        plt.savefig(save_path)
    else:
        plt.close()

plot_error_boxplot(benchmark_results, save_path=None)

Boxplot of error rates for each oversampler.

Parameters:

Name	Type	Description	Default
benchmark_results	DataFrame	Benchmark results dataframe.	required
save_path	str | None	Optional output image path.	None

Source code in src/oversampleqa/plotting.py

def plot_error_boxplot(
    benchmark_results: pd.DataFrame, save_path: str | None = None
) -> None:
    """Boxplot of error rates for each oversampler.

    Args:
        benchmark_results: Benchmark results dataframe.
        save_path: Optional output image path.
    """
    benchmark_results.boxplot(column="error_rate", by="oversampler")
    plt.ylabel("Error rate")
    plt.title("Error rate distribution")
    plt.suptitle("")
    plt.tight_layout()
    if save_path:
        plt.savefig(save_path)
    plt.close()

plot_error_comparison(benchmark_results, save_path=None)

Bar plot showing mean error rates for each oversampler.

Parameters:

Name	Type	Description	Default
benchmark_results	DataFrame	Benchmark results dataframe.	required
save_path	str | None	Optional output image path.	None

Source code in src/oversampleqa/plotting.py

def plot_error_comparison(
    benchmark_results: pd.DataFrame, save_path: str | None = None
) -> None:
    """Bar plot showing mean error rates for each oversampler.

    Args:
        benchmark_results: Benchmark results dataframe.
        save_path: Optional output image path.
    """
    summary = benchmark_results.groupby("oversampler")["error_rate"].mean()
    summary.plot(kind="bar")
    plt.ylabel("Mean error rate")
    if save_path:
        plt.savefig(save_path)
    plt.close()

plot_error_heatmap(error_matrix, class_labels=None, save_path=None)

Plot heatmap of a multi-class error attribution matrix.

Parameters

error_matrix : ndarray Matrix where matrix[i, j] counts synthetic samples generated for class i that are closest to hidden samples from class j. class_labels : list of int, optional Labels for the classes corresponding to the rows/columns of the matrix. If not provided, integer indices are used. save_path : str, optional If given, path to save the resulting plot. Otherwise the figure is closed and not displayed.

Source code in src/oversampleqa/plotting.py

def plot_error_heatmap(
    error_matrix: NDArray[np.integer],
    class_labels: list[int] | None = None,
    save_path: str | None = None,
) -> None:
    """Plot heatmap of a multi-class error attribution matrix.

    Parameters
    ----------
    error_matrix : ndarray
        Matrix where ``matrix[i, j]`` counts synthetic samples generated for
        class ``i`` that are closest to hidden samples from class ``j``.
    class_labels : list of int, optional
        Labels for the classes corresponding to the rows/columns of the matrix.
        If not provided, integer indices are used.
    save_path : str, optional
        If given, path to save the resulting plot. Otherwise the figure is
        closed and not displayed.
    """

    labels = (
        class_labels if class_labels is not None else list(range(len(error_matrix)))
    )
    df = pd.DataFrame(error_matrix, index=labels, columns=labels)
    plt.figure()
    sns.heatmap(df, annot=True, fmt="d", cmap="Blues")
    plt.xlabel("Hidden class")
    plt.ylabel("Synthetic class")
    plt.tight_layout()
    if save_path:
        plt.savefig(save_path)
    plt.close()

plot_error_ranking(benchmark_results, save_path=None)

Line chart of mean error rate ranked by oversampler.

Parameters:

Name	Type	Description	Default
benchmark_results	DataFrame	Benchmark results dataframe.	required
save_path	str | None	Optional output image path.	None

Source code in src/oversampleqa/plotting.py

def plot_error_ranking(
    benchmark_results: pd.DataFrame, save_path: str | None = None
) -> None:
    """Line chart of mean error rate ranked by oversampler.

    Args:
        benchmark_results: Benchmark results dataframe.
        save_path: Optional output image path.
    """
    summary = (
        benchmark_results.groupby("oversampler")["error_rate"].mean().sort_values()
    )
    plt.figure()
    plt.plot(range(1, len(summary) + 1), summary.values, marker="o")
    plt.xticks(range(1, len(summary) + 1), summary.index, rotation=45, ha="right")
    plt.xlabel("Rank (lower is better)")
    plt.ylabel("Mean error rate")
    plt.tight_layout()
    if save_path:
        plt.savefig(save_path)
    plt.close()

plot_noise_sensitivity(results, save_path=None)

Line plot showing error rate as label noise increases.

Parameters

results : DataFrame Output of :func:oversampleqa.metrics.noise_sensitivity_diagnostic, expected to contain noise and error_rate columns. save_path : str, optional If given, path to save the resulting plot. Otherwise the figure is closed and not displayed.

Source code in src/oversampleqa/plotting.py

def plot_noise_sensitivity(results: pd.DataFrame, save_path: str | None = None) -> None:
    """Line plot showing error rate as label noise increases.

    Parameters
    ----------
    results : DataFrame
        Output of :func:`oversampleqa.metrics.noise_sensitivity_diagnostic`,
        expected to contain ``noise`` and ``error_rate`` columns.
    save_path : str, optional
        If given, path to save the resulting plot. Otherwise the figure is
        closed and not displayed.
    """

    plt.figure()
    sns.lineplot(data=results, x="noise", y="error_rate", marker="o")
    plt.xlabel("Label noise")
    plt.ylabel("Error rate")
    plt.tight_layout()
    if save_path:
        plt.savefig(save_path)
    plt.close()

plot_sample_distribution(majority, minority, synthetic, hidden_majority=None, method='pca', save_path=None)

Visualize sample distribution using PCA or UMAP.

Parameters

majority, minority, synthetic : ndarray Arrays of majority, minority and synthetic samples. hidden_majority : ndarray, optional Hidden majority samples for reference. method : {{"pca", "umap"}}, default="pca" Dimensionality reduction method to use. save_path : str, optional If given, path to save the resulting plot. Otherwise the figure is closed and not displayed.

Source code in src/oversampleqa/plotting.py

def plot_sample_distribution(
    majority: NDArray[np.floating],
    minority: NDArray[np.floating],
    synthetic: NDArray[np.floating],
    hidden_majority: NDArray[np.floating] | None = None,
    method: str = "pca",
    save_path: str | None = None,
) -> None:
    """Visualize sample distribution using PCA or UMAP.

    Parameters
    ----------
    majority, minority, synthetic : ndarray
        Arrays of majority, minority and synthetic samples.
    hidden_majority : ndarray, optional
        Hidden majority samples for reference.
    method : {{"pca", "umap"}}, default="pca"
        Dimensionality reduction method to use.
    save_path : str, optional
        If given, path to save the resulting plot. Otherwise the figure is
        closed and not displayed.
    """

    if method not in {"pca", "umap"}:
        raise ValueError("method must be 'pca' or 'umap'")

    X = np.vstack([majority, minority, synthetic])
    if method == "pca":
        reducer = PCA(n_components=2)
    else:
        if UMAP is None:
            raise ImportError("umap-learn is required for method='umap'")
        reducer = UMAP(n_components=2, random_state=42, n_jobs=1)

    comps = reducer.fit_transform(X)
    n_maj = len(majority)
    n_min = len(minority)

    plt.figure()
    plt.scatter(comps[:n_maj, 0], comps[:n_maj, 1], label="majority", alpha=0.5)
    plt.scatter(
        comps[n_maj : n_maj + n_min, 0],
        comps[n_maj : n_maj + n_min, 1],
        label="minority",
        alpha=0.5,
    )
    plt.scatter(
        comps[n_maj + n_min :, 0],
        comps[n_maj + n_min :, 1],
        label="synthetic",
        alpha=0.5,
    )

    if hidden_majority is not None:
        hid_comps = reducer.transform(hidden_majority)
        plt.scatter(
            hid_comps[:, 0], hid_comps[:, 1], label="hidden majority", marker="x"
        )

    plt.legend()
    if save_path:
        plt.savefig(save_path)
    plt.close()

check_metric_axioms(func, name='metric', *, domain='real', n_trials=50, n_features=4, tolerance=1e-09, random_state=0, **metric_kwargs)

Check that a callable behaves like a distance metric.

Checks, on random vectors:

identity d(x, x) == 0. identity_of_indiscernibles d(x, y) > 0 whenever x != y. This is the check the built-in Hassanat implementation failed -- it scored [-5] against [5] as zero, because it compared absolute values. symmetry d(x, y) == d(y, x). non_negativity d(x, y) >= 0. finiteness No nan or inf on ordinary input.

The triangle inequality is deliberately not checked: several useful registry entries are genuine semi-metrics, so requiring it would reject metrics the package intends to support. Identity of indiscernibles is the one whose violation makes a metric silently meaningless.

Parameters:

Name	Type	Description	Default
func	Any	Candidate metric.	required
name	str	Name used in failure messages.	'metric'
domain	MetricDomain	Input the metric is defined on. "sample" metrics compare distributions rather than points, so the point-metric axioms are skipped for them. See :data:METRIC_DOMAINS.	'real'
n_trials	int	Random vector pairs to test.	50
n_features	int	Dimension of the test vectors.	4
tolerance	float	Numerical slack.	1e-09
random_state	int	Seed, so failures reproduce.	0
**metric_kwargs	Any	Extra arguments forwarded to the metric.	{}

Returns:

Type	Description
AxiomReport	AxiomReport, falsy when any axiom failed.

Source code in src/oversampleqa/plugin_contract.py

def check_metric_axioms(
    func: Any,
    name: str = "metric",
    *,
    domain: MetricDomain = "real",
    n_trials: int = 50,
    n_features: int = 4,
    tolerance: float = 1e-9,
    random_state: int = 0,
    **metric_kwargs: Any,
) -> AxiomReport:
    """Check that a callable behaves like a distance metric.

    Checks, on random vectors:

    ``identity``
        ``d(x, x) == 0``.
    ``identity_of_indiscernibles``
        ``d(x, y) > 0`` whenever ``x != y``. **This is the check the built-in
        Hassanat implementation failed** -- it scored ``[-5]`` against ``[5]``
        as zero, because it compared absolute values.
    ``symmetry``
        ``d(x, y) == d(y, x)``.
    ``non_negativity``
        ``d(x, y) >= 0``.
    ``finiteness``
        No ``nan`` or ``inf`` on ordinary input.

    The triangle inequality is deliberately **not** checked: several useful
    registry entries are genuine semi-metrics, so requiring it would reject
    metrics the package intends to support. Identity of indiscernibles is the
    one whose violation makes a metric silently meaningless.

    Args:
        func: Candidate metric.
        name: Name used in failure messages.
        domain: Input the metric is defined on. ``"sample"`` metrics compare
            distributions rather than points, so the point-metric axioms are
            skipped for them. See :data:`METRIC_DOMAINS`.
        n_trials: Random vector pairs to test.
        n_features: Dimension of the test vectors.
        tolerance: Numerical slack.
        random_state: Seed, so failures reproduce.
        **metric_kwargs: Extra arguments forwarded to the metric.

    Returns:
        AxiomReport, falsy when any axiom failed.
    """
    if domain == "sample":
        # Sample-based metrics answer a different question -- they compare two
        # sets of observations, not two points -- so identity of indiscernibles
        # is not even meaningful for them.
        return AxiomReport(True, True, True, True, True, ())

    rng = np.random.default_rng(random_state)
    failures: list[str] = []

    def draw() -> NDArray[np.floating]:
        if domain == "non_negative":
            return rng.random(n_features) + 0.1
        if domain == "boolean":
            return (rng.random(n_features) < 0.5).astype(float)
        return rng.normal(0, 5, size=n_features)

    identity = True
    indiscernibles = True
    symmetry = True
    non_negative = True
    finite = True

    for _ in range(n_trials):
        x = draw()
        y = draw()
        if domain == "boolean" and np.array_equal(x, y):
            continue  # boolean draws collide; that is not a violation

        try:
            d_xy = float(func(x, y, **metric_kwargs))
            d_yx = float(func(y, x, **metric_kwargs))
            d_xx = float(func(x, x, **metric_kwargs))
        except Exception as exc:
            failures.append(f"raised {type(exc).__name__}: {exc}")
            return AxiomReport(False, False, False, False, False, tuple(failures))

        if not (np.isfinite(d_xy) and np.isfinite(d_xx)):
            finite = False
        if abs(d_xx) > tolerance:
            identity = False
        if d_xy < -tolerance:
            non_negative = False
        if abs(d_xy - d_yx) > tolerance:
            symmetry = False
        if d_xy <= tolerance:
            # Random continuous vectors are distinct with probability 1.
            indiscernibles = False

    # The specific case the broken Hassanat passed everything else on. Only
    # meaningful where sign carries information: a boolean set metric is
    # *supposed* to map -5 and 5 to the same element, and a non-negative domain
    # has no mirrored pair.
    mirrored = np.full(n_features, 5.0)
    if domain != "real":
        return AxiomReport(
            identity=identity,
            identity_of_indiscernibles=indiscernibles,
            symmetry=symmetry,
            non_negativity=non_negative,
            finiteness=finite,
            failures=tuple(
                f"{name}: {f}"
                for f in _collect(
                    identity, indiscernibles, symmetry, non_negative, finite, failures
                )
            ),
        )
    try:
        d_mirror = float(func(-mirrored, mirrored, **metric_kwargs))
        if abs(d_mirror) <= tolerance:
            indiscernibles = False
            failures.append(
                "d(-x, x) == 0 for x = 5: distinct points at distance zero. "
                "This usually means the metric compares magnitudes and discards "
                "sign -- the exact defect found in the original hassanat "
                "implementation."
            )
    except Exception:
        pass

    failures = _collect(
        identity, indiscernibles, symmetry, non_negative, finite, failures
    )

    return AxiomReport(
        identity=identity,
        identity_of_indiscernibles=indiscernibles,
        symmetry=symmetry,
        non_negativity=non_negative,
        finiteness=finite,
        failures=tuple(f"{name}: {f}" for f in failures),
    )

register_metric(name)

Decorator to register a metric plugin by name.

Parameters:

Name	Type	Description	Default
name	str	Metric identifier.	required

Source code in src/oversampleqa/plugin_system.py

def register_metric(name: str) -> Callable[[Any], Any]:
    """Decorator to register a metric plugin by name.

    Args:
        name: Metric identifier.
    """

    def decorator(cls: type[DistanceMetricProtocol]) -> type[DistanceMetricProtocol]:
        plugin_manager.register_metric(name, cls)
        return cls

    return decorator

register_validator(name)

Decorator to register a validator plugin by name.

Parameters:

Name	Type	Description	Default
name	str	Validator identifier.	required

Source code in src/oversampleqa/plugin_system.py

def register_validator(name: str) -> Callable[[Any], Any]:
    """Decorator to register a validator plugin by name.

    Args:
        name: Validator identifier.
    """

    def decorator(cls: type[ValidatorProtocol]) -> type[ValidatorProtocol]:
        plugin_manager.register_validator(name, cls)
        return cls

    return decorator

generate_report(benchmark_results, output_format='markdown', output_path=None, include_plots=True, fidelity_reports=None)

Generate a report from benchmark results.

Parameters:

Name	Type	Description	Default
benchmark_results	DataFrame	Benchmark results dataframe.	required
output_format	str	Output format (markdown or html).	'markdown'
output_path	str | None	Optional output file path.	None
include_plots	bool	Whether to include plot artifacts.	True
fidelity_reports	dict[str, Any] | None	Optional mapping of oversampler name to :class:~oversampleqa.fidelity.FidelityReport. When given, a fidelity section is appended covering the axis the error rate cannot express.	None

Returns:

Type	Description
str	Rendered report content as a string.

Raises:

Type	Description
ValueError	If output_format is not recognised.

Source code in src/oversampleqa/report.py

def generate_report(
    benchmark_results: pd.DataFrame,
    output_format: str = "markdown",
    output_path: str | None = None,
    include_plots: bool = True,
    fidelity_reports: dict[str, Any] | None = None,
) -> str:
    """Generate a report from benchmark results.

    Args:
        benchmark_results: Benchmark results dataframe.
        output_format: Output format (``markdown`` or ``html``).
        output_path: Optional output file path.
        include_plots: Whether to include plot artifacts.
        fidelity_reports: Optional mapping of oversampler name to
            :class:`~oversampleqa.fidelity.FidelityReport`. When given, a
            fidelity section is appended covering the axis the error rate
            cannot express.

    Returns:
        Rendered report content as a string.

    Raises:
        ValueError: If ``output_format`` is not recognised.
    """
    if output_format not in {"markdown", "html"}:
        raise ValueError("output_format must be 'markdown' or 'html'")

    summary = compute_ranking(benchmark_results)
    if output_format == "markdown":
        content = "\n".join(
            [
                "# OversampleQA Report",
                "",
                "## Run metadata",
                "",
                report_metadata_markdown(benchmark_results),
                "",
                "## Ranking",
                "",
                frame_to_markdown(summary),
            ]
        )
    else:
        content = (
            "<h1>OversampleQA Report</h1><h2>Run metadata</h2>"
            + report_metadata_html(benchmark_results)
            + "<h2>Ranking</h2>"
            + summary.to_html()
        )

    if fidelity_reports:
        content += _fidelity_section(fidelity_reports, output_format)

    if include_plots and output_path:
        base = str(output_path).rsplit(".", 1)[0]
        box_path = base + "_box.png"
        rank_path = base + "_rank.png"
        plot_error_boxplot(benchmark_results, save_path=box_path)
        plot_error_ranking(benchmark_results, save_path=rank_path)
        if output_format == "markdown":
            content += f"\n\n![boxplot]({box_path})\n![ranking]({rank_path})\n"

    if output_path:
        with open(output_path, "w", encoding="utf-8") as f:
            f.write(content)
        write_export_metadata(
            output_path,
            export_kind="benchmark_report",
            data=summary,
            extra={
                "source": {
                    "row_count": len(benchmark_results),
                    "columns": [str(column) for column in benchmark_results.columns],
                    "attrs": dict(benchmark_results.attrs),
                }
            },
        )
    return content

evaluate_surrogate_models(X, y, minority_label, oversampler, model, test_size=0.3, random_state=None)

Evaluate model performance with and without synthetic data.

The function trains the provided model under three scenarios:

1. real_only – using the original training data without oversampling.
2. real_plus_synth – using the oversampled training data.
3. synth_only – replacing the real minority samples with the synthetic samples generated by the oversampler.

Parameters

X, y : ndarray Input data and labels. minority_label : int Label of the minority class. oversampler : imblearn BaseOverSampler Oversampler instance used to generate synthetic samples. model : sklearn estimator Classifier implementing fit/predict. test_size : float, default=0.3 Fraction of the dataset reserved for testing. random_state : int, optional Random seed for the split.

Returns

dict Mapping of scenario names to dictionaries with f1, recall and precision scores.

Source code in src/oversampleqa/surrogate.py

def evaluate_surrogate_models(
    X: NDArray[np.floating],
    y: NDArray[Any],
    minority_label: int,
    oversampler: Any,
    model: Any,
    test_size: float = 0.3,
    random_state: int | None = None,
) -> dict[str, dict[str, float]]:
    """Evaluate model performance with and without synthetic data.

    The function trains the provided ``model`` under three scenarios:

    1. ``real_only`` – using the original training data without oversampling.
    2. ``real_plus_synth`` – using the oversampled training data.
    3. ``synth_only`` – replacing the real minority samples with the synthetic
       samples generated by the oversampler.

    Parameters
    ----------
    X, y : ndarray
        Input data and labels.
    minority_label : int
        Label of the minority class.
    oversampler : imblearn BaseOverSampler
        Oversampler instance used to generate synthetic samples.
    model : sklearn estimator
        Classifier implementing ``fit``/``predict``.
    test_size : float, default=0.3
        Fraction of the dataset reserved for testing.
    random_state : int, optional
        Random seed for the split.

    Returns
    -------
    dict
        Mapping of scenario names to dictionaries with ``f1``, ``recall`` and
        ``precision`` scores.
    """

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=test_size, stratify=y, random_state=random_state
    )

    # Scenario 1: real-only
    model_real = clone(model)
    model_real.fit(X_train, y_train)
    pred_real = model_real.predict(X_test)

    # Scenario 2: real + synthetic
    oversampler_plus = clone(oversampler)
    try:
        X_res, y_res = oversampler_plus.fit_resample(X_train, y_train)
    except Exception:  # pragma: no cover - defensive
        logger.exception("Oversampler failed during fit_resample")
        raise
    model_plus = clone(model)
    model_plus.fit(X_res, y_res)
    pred_plus = model_plus.predict(X_test)

    # Scenario 3: synthetic-only
    synthetic = extract_synthetic_samples(X_train, X_res, y_res, minority_label)
    majority_mask = y_train != minority_label
    X_majority = X_train[majority_mask]
    y_majority = y_train[majority_mask]
    if len(synthetic) == 0:
        X_syn = X_train
        y_syn = y_train
    else:
        X_syn = np.vstack([X_majority, synthetic])
        y_syn = np.hstack([y_majority, np.full(len(synthetic), minority_label)])
    model_syn = clone(model)
    model_syn.fit(X_syn, y_syn)
    pred_syn = model_syn.predict(X_test)

    def _scores(y_true: NDArray[Any], y_pred: NDArray[Any]) -> dict[str, float]:
        mask_true = y_true == minority_label
        mask_pred = y_pred == minority_label
        return {
            "f1": f1_score(mask_true, mask_pred),
            "recall": recall_score(mask_true, mask_pred),
            "precision": precision_score(mask_true, mask_pred),
        }

    return {
        "real_only": _scores(y_test, pred_real),
        "real_plus_synth": _scores(y_test, pred_plus),
        "synth_only": _scores(y_test, pred_syn),
    }

validation_session(config) async

Async context manager that yields a TypedValidator.

Parameters:

Name	Type	Description	Default
config	ValidationConfig	ValidationConfig (reserved for future use).	required

Yields:

Type	Description
AsyncIterator[TypedValidator]	TypedValidator instance.

Source code in src/oversampleqa/typed_validator.py

@asynccontextmanager
async def validation_session(config: ValidationConfig) -> AsyncIterator[TypedValidator]:
    """Async context manager that yields a TypedValidator.

    Args:
        config: ValidationConfig (reserved for future use).

    Yields:
        TypedValidator instance.
    """
    validator = TypedValidator()
    try:
        yield validator
    finally:
        # No `return` here. A bare return inside `finally` swallows whatever
        # exception was in flight, so a failure inside the session body
        # disappeared silently and the caller saw a clean exit.
        logger.debug("validation_session closed")

extract_synthetic_samples(X_original, X_resampled, y_resampled, minority_label)

Return synthetic minority samples from a resampled dataset.

Parameters

X_original : ndarray Original feature matrix used for fitting the oversampler. X_resampled : ndarray Feature matrix returned by oversampler.fit_resample. y_resampled : ndarray Corresponding labels for X_resampled. minority_label : int Label of the minority class that was oversampled.

Returns

ndarray Array containing only the synthetic minority samples.

Raises

ValueError If the oversampler did not preserve the original samples as a prefix of its output, so synthetic rows cannot be identified positionally.

Notes

Synthetic samples are identified positionally: everything after the original rows is assumed to be new. That holds for the SMOTE family and RandomOverSampler, which append. It does not hold for combined over/under-samplers such as SMOTEENN and SMOTETomek, which delete original rows. A length check alone does not catch this -- SMOTEENN can still return more rows than it was given while having removed some of the originals -- so the prefix is compared element-wise.

Source code in src/oversampleqa/validator.py

def extract_synthetic_samples(
    X_original: NDArray[np.floating],
    X_resampled: NDArray[np.floating],
    y_resampled: NDArray[np.integer],
    minority_label: int,
) -> NDArray[np.floating]:
    """Return synthetic minority samples from a resampled dataset.

    Parameters
    ----------
    X_original : ndarray
        Original feature matrix used for fitting the oversampler.
    X_resampled : ndarray
        Feature matrix returned by ``oversampler.fit_resample``.
    y_resampled : ndarray
        Corresponding labels for ``X_resampled``.
    minority_label : int
        Label of the minority class that was oversampled.

    Returns
    -------
    ndarray
        Array containing only the synthetic minority samples.

    Raises
    ------
    ValueError
        If the oversampler did not preserve the original samples as a prefix
        of its output, so synthetic rows cannot be identified positionally.

    Notes
    -----
    Synthetic samples are identified positionally: everything after the
    original rows is assumed to be new. That holds for the SMOTE family and
    ``RandomOverSampler``, which append. It does **not** hold for combined
    over/under-samplers such as ``SMOTEENN`` and ``SMOTETomek``, which delete
    original rows. A length check alone does not catch this -- ``SMOTEENN``
    can still return more rows than it was given while having removed some of
    the originals -- so the prefix is compared element-wise.
    """
    require_prefix_preserved(X_original, X_resampled, "validate_oversampling")
    n = len(X_original)
    synthetic: NDArray[np.floating] = X_resampled[n:][y_resampled[n:] == minority_label]
    return synthetic

validate_multiclass_oversampling(X, y, oversampler, hidden_ratio=0.1, metric='hassanat', metric_kwargs=None, return_matrix=False, *, min_hidden=5, random_state=42)

Validate oversampling for multi-class datasets.

For each class label, a portion of samples is hidden and the remainder is used for training the oversampler along with all other visible classes. The nearest hidden class to each synthetic sample determines the error attribution. The function returns the per-class error rates and optionally the full error matrix where matrix[i, j] counts how many synthetic samples generated for class i are closest to hidden samples from class j.

Parameters

X, y : ndarray Input data and labels. oversampler : BaseOverSampler Instance of an imbalanced-learn oversampler supporting multi-class data. hidden_ratio : float, default=0.1 Fraction of each class to hide during validation. metric : str, default="hassanat" Distance metric to use. metric_kwargs : dict, optional Additional keyword arguments passed to :func:distance_matrix. return_matrix : bool, default=False If True also return the error matrix. min_hidden : int, default=5 Minimum held-out points per class. Every class is a reference for every other, so a class that cannot supply a usable hidden set makes attribution unreliable for all of them. random_state : int, Generator, SeedSequence or None, default=42 Seeds the per-class hold-out. Defaults to 42, which reproduces previously documented numbers.

Returns

dict or tuple Mapping of class_label -> error_rate. If return_matrix is True the second element is the error matrix.

A class for which the sampler generated no synthetic points maps to
``nan``, not ``0.0``: nothing was measured, and ``0.0`` is the score of
a perfect result. Use :func:`macro_error_rate` to summarise, or
``np.nanmean`` over the values -- a plain mean propagates the ``nan``.

Raises

ValueError If hidden_ratio is out of range, if any class would hide fewer than min_hidden points, or if the oversampler did not preserve the original rows as a prefix of its output.

Notes

A synthetic point equidistant from its own class and another is attributed to its own class, matching :func:score_nearest_distances. Ties previously went to whichever class appeared first in label order, which biased results toward low-numbered classes for reasons unrelated to the data.

Source code in src/oversampleqa/validator.py

def validate_multiclass_oversampling(
    X: NDArray[np.floating],
    y: NDArray[np.integer],
    oversampler: BaseOverSampler,
    hidden_ratio: float = 0.1,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
    return_matrix: bool = False,
    *,
    min_hidden: int = 5,
    random_state: RandomStateLike = 42,
) -> dict[int, float] | tuple[dict[int, float], NDArray[np.floating]]:
    """Validate oversampling for multi-class datasets.

    For each class label, a portion of samples is hidden and the remainder
    is used for training the oversampler along with all other visible
    classes. The nearest hidden class to each synthetic sample determines
    the error attribution. The function returns the per-class error rates
    and optionally the full error matrix where ``matrix[i, j]`` counts how
    many synthetic samples generated for class ``i`` are closest to hidden
    samples from class ``j``.

    Parameters
    ----------
    X, y : ndarray
        Input data and labels.
    oversampler : BaseOverSampler
        Instance of an `imbalanced-learn` oversampler supporting
        multi-class data.
    hidden_ratio : float, default=0.1
        Fraction of each class to hide during validation.
    metric : str, default="hassanat"
        Distance metric to use.
    metric_kwargs : dict, optional
        Additional keyword arguments passed to :func:`distance_matrix`.
    return_matrix : bool, default=False
        If ``True`` also return the error matrix.
    min_hidden : int, default=5
        Minimum held-out points per class. Every class is a reference for
        every other, so a class that cannot supply a usable hidden set makes
        attribution unreliable for all of them.
    random_state : int, Generator, SeedSequence or None, default=42
        Seeds the per-class hold-out. Defaults to 42, which reproduces
        previously documented numbers.

    Returns
    -------
    dict or tuple
        Mapping of ``class_label -> error_rate``. If ``return_matrix`` is
        ``True`` the second element is the error matrix.

        A class for which the sampler generated no synthetic points maps to
        ``nan``, not ``0.0``: nothing was measured, and ``0.0`` is the score of
        a perfect result. Use :func:`macro_error_rate` to summarise, or
        ``np.nanmean`` over the values -- a plain mean propagates the ``nan``.

    Raises
    ------
    ValueError
        If ``hidden_ratio`` is out of range, if any class would hide fewer than
        ``min_hidden`` points, or if the oversampler did not preserve the
        original rows as a prefix of its output.

    Notes
    -----
    A synthetic point equidistant from its own class and another is attributed
    to its own class, matching :func:`score_nearest_distances`. Ties previously
    went to whichever class appeared first in label order, which biased results
    toward low-numbered classes for reasons unrelated to the data.
    """

    _validate_hidden_ratio(hidden_ratio)
    require_pointwise_metric(metric)
    labels = np.unique(y)
    rng = as_generator(random_state)

    # Every class supplies a hidden reference, so every class must be able to
    # support one. Previously a class whose hold-out rounded to zero got an
    # empty reference and was silently dropped from attribution: synthetic
    # points could never be scored against it, so it could never receive an
    # error, and its own rate was computed against a reduced set of rivals.
    too_small = {
        int(lbl): int(np.sum(y == lbl) * hidden_ratio)
        for lbl in labels
        if int(np.sum(y == lbl) * hidden_ratio) < min_hidden
    }
    if too_small:
        detail = ", ".join(
            f"class {lbl} would hide {n}" for lbl, n in sorted(too_small.items())
        )
        raise ValueError(
            f"Hiding {hidden_ratio:.3g} of each class leaves fewer than "
            f"min_hidden={min_hidden} points for: {detail}. A nearest-neighbour "
            "comparison against so few points is not meaningful, and every class "
            "is used as a reference for every other. Supply more data for those "
            "classes, raise hidden_ratio, or lower min_hidden."
        )

    visible = {}
    hidden = {}
    for label in labels:
        cls_samples = X[y == label]
        n_hidden = int(len(cls_samples) * hidden_ratio)
        idx = rng.permutation(len(cls_samples))
        hidden[label] = cls_samples[idx[:n_hidden]]
        visible[label] = cls_samples[idx[n_hidden:]]

    X_train = np.vstack([visible[lbl] for lbl in labels])
    y_train = np.hstack([[lbl] * len(visible[lbl]) for lbl in labels])

    try:
        X_res, y_res = oversampler.fit_resample(X_train, y_train)
    except Exception:  # pragma: no cover - defensive
        logger.exception("Oversampler failed during fit_resample")
        raise

    # The binary path has always checked this; the multiclass path did not, so
    # a combined sampler that deletes original rows produced a misaligned slice
    # and plausible-looking numbers from it.
    require_prefix_preserved(X_train, X_res, "validate_multiclass_oversampling")

    start = len(X_train)
    X_syn = X_res[start:]
    y_syn = y_res[start:]

    metric_kwargs = metric_kwargs or {}
    matrix = np.zeros((len(labels), len(labels)), dtype=int)

    # Distance from every synthetic point to each class's hidden reference.
    hidden_dists = {
        lbl: distance_matrix(X_syn, hidden[lbl], metric, **metric_kwargs)
        for lbl in labels
    }

    for i, lbl in enumerate(labels):
        rows = y_syn == lbl
        if not np.any(rows):
            continue
        # Nearest distance to each class, as (n_synthetic, n_classes).
        per_class = np.column_stack(
            [hidden_dists[other][rows].min(axis=1) for other in labels]
        )
        own = per_class[:, i]
        rivals = np.delete(per_class, i, axis=1)
        best_rival = rivals.min(axis=1)

        # A tie is attributed to the point's own class. Strict `<` on a running
        # minimum previously gave ties to whichever class came first in label
        # order, a systematic bias toward low-numbered classes that had nothing
        # to do with the data. This matches score_nearest_distances, where a tie
        # is not evidence of an error.
        own_wins = own <= best_rival
        rival_idx = np.argmin(rivals, axis=1)
        # argmin indexes the array with column i removed; shift back past it.
        rival_idx[rival_idx >= i] += 1

        attribution = np.where(own_wins, i, rival_idx)
        for j in range(len(labels)):
            matrix[i, j] = int(np.sum(attribution == j))

    error_rates = {}
    for i, lbl in enumerate(labels):
        n_syn = int(matrix[i].sum())
        if n_syn == 0:
            # nan, not 0.0. The sampler generated nothing for this class, so
            # nothing was measured -- and 0.0 is the score of a perfect result.
            error_rates[int(lbl)] = float("nan")
            continue
        errors = n_syn - int(matrix[i, i])
        error_rates[int(lbl)] = calculate_error_rate(errors, n_syn)

    if return_matrix:
        return error_rates, matrix

    return error_rates

validate_oversampling(X, y, minority_label, oversampler, hidden_ratio=0.1, metric='hassanat', metric_kwargs=None, return_details=False, *, reference='hidden_minority', minority_hidden_ratio=None, min_hidden=5, duplication_warn_threshold=0.5, random_state=42, stratify_by=None, n_repeats=1, reseed_oversampler=False)

Validate oversampling using the hidden majority approach.

A fraction of the majority class is held out. The oversampler is fitted on what remains, and each synthetic minority point is scored by comparing its nearest-neighbour distance to the hidden majority against its nearest-neighbour distance to a minority reference set. A synthetic point strictly closer to the hidden majority counts as an error.

Parameters

X, y : ndarray Input data and labels. minority_label : int Label of the minority class. oversampler : BaseOverSampler Instance of an imbalanced-learn oversampler. hidden_ratio : float, default=0.1 Fraction of majority samples to hide during validation. metric : str, default="hassanat" Distance metric to use. metric_kwargs : dict, optional Additional keyword arguments passed to :func:distance_matrix. return_details : bool, default=False If True return a :class:~oversampleqa.types.ValidationDetails instead of the bare error rate. reference : {"hidden_minority", "train_minority"}, default="hidden_minority" Which minority set the synthetic points are compared against.

``"hidden_minority"`` also holds out part of the minority class and
compares against those held-out points. Both sides of the comparison
are then unseen, which is the same estimand
:func:`validate_multiclass_oversampling` uses.

``"train_minority"`` compares against the full minority class -- the
very data the oversampler interpolated from. This is the historical
behaviour, retained so old numbers can be reproduced. It biases the
result toward "no error" by an amount that depends on minority
density rather than on oversampler quality, and emits a
``FutureWarning``.

minority_hidden_ratio : float, optional Fraction of the minority class to hide when reference="hidden_minority". Defaults to hidden_ratio. min_hidden : int, default=5 Minimum number of held-out minority points required. A nearest-neighbour comparison against fewer than a handful of points is not meaningful, so this raises rather than warns. duplication_warn_threshold : float, default=0.5 Emit a UserWarning when this fraction of synthetic points coincide with a real point. random_state : int, Generator, SeedSequence or None, default=42 Seeds the hold-out split. Which points get hidden is the single largest driver of the error rate, so varying this varies the result. The default of 42 reproduces previously documented numbers; None draws fresh entropy and is not reproducible. stratify_by : ndarray, optional Group labels aligned with y. When given, the majority hold-out takes hidden_ratio within each group, so a hold-out cannot miss a cluster entirely. Strata are never inferred automatically. n_repeats : int, default=1 Number of independent hold-out splits. Above 1, the returned details carry the per-repeat vector and its dispersion. Repeat streams are spawned from a SeedSequence rather than derived as seed + i, which would correlate them. reseed_oversampler : bool, default=False Give the oversampler a fresh seed on each repeat. This changes what the dispersion covers -- see Notes.

Returns

float or ValidationDetails Error rate by default, nan if no synthetic samples were produced. With n_repeats > 1 the bare return is the mean across repeats.

Raises

ValueError If the labels are not binary, if minority_label is absent, if the held-out minority would be smaller than min_hidden, or if the oversampler does not preserve the original samples as a prefix.

Notes

The error rate is a relative quantity. Its scale depends on hidden_ratio, on the density of the data, and on dimensionality, so values are not comparable across datasets. See :doc:/concepts.

What the repeat interval covers. With n_repeats > 1 the reported interval is a percentile bootstrap over the per-repeat error rates. It describes the variability of the hold-out split, conditional on this dataset and on the oversampler's own seed. It is not a confidence interval for a population quantity, and with reseed_oversampler=False it does not include the oversampler's own randomness at all. Setting reseed_oversampler=True clones the sampler with a fresh seed per repeat, so the dispersion then covers both sources together -- a different, wider decomposition.

Synthetic points generated from shared parent points are not independent, so a binomial interval on the error rate would be too narrow. Nothing here claims more than the repeat-level bootstrap supports.

Source code in src/oversampleqa/validator.py

def validate_oversampling(
    X: NDArray[np.floating],
    y: NDArray[np.integer],
    minority_label: int,
    oversampler: BaseOverSampler,
    hidden_ratio: float = 0.1,
    metric: str = "hassanat",
    metric_kwargs: dict[str, Any] | None = None,
    return_details: bool = False,
    *,
    reference: ReferenceSet = "hidden_minority",
    minority_hidden_ratio: float | None = None,
    min_hidden: int = 5,
    duplication_warn_threshold: float = 0.5,
    random_state: RandomStateLike = 42,
    stratify_by: NDArray[Any] | None = None,
    n_repeats: int = 1,
    reseed_oversampler: bool = False,
) -> float | ValidationDetails:
    """Validate oversampling using the hidden majority approach.

    A fraction of the majority class is held out. The oversampler is fitted on
    what remains, and each synthetic minority point is scored by comparing its
    nearest-neighbour distance to the hidden majority against its
    nearest-neighbour distance to a minority reference set. A synthetic point
    strictly closer to the hidden majority counts as an error.

    Parameters
    ----------
    X, y : ndarray
        Input data and labels.
    minority_label : int
        Label of the minority class.
    oversampler : BaseOverSampler
        Instance of an `imbalanced-learn` oversampler.
    hidden_ratio : float, default=0.1
        Fraction of majority samples to hide during validation.
    metric : str, default="hassanat"
        Distance metric to use.
    metric_kwargs : dict, optional
        Additional keyword arguments passed to :func:`distance_matrix`.
    return_details : bool, default=False
        If ``True`` return a :class:`~oversampleqa.types.ValidationDetails`
        instead of the bare error rate.
    reference : {"hidden_minority", "train_minority"}, default="hidden_minority"
        Which minority set the synthetic points are compared against.

        ``"hidden_minority"`` also holds out part of the minority class and
        compares against those held-out points. Both sides of the comparison
        are then unseen, which is the same estimand
        :func:`validate_multiclass_oversampling` uses.

        ``"train_minority"`` compares against the full minority class -- the
        very data the oversampler interpolated from. This is the historical
        behaviour, retained so old numbers can be reproduced. It biases the
        result toward "no error" by an amount that depends on minority
        density rather than on oversampler quality, and emits a
        ``FutureWarning``.
    minority_hidden_ratio : float, optional
        Fraction of the minority class to hide when
        ``reference="hidden_minority"``. Defaults to ``hidden_ratio``.
    min_hidden : int, default=5
        Minimum number of held-out minority points required. A
        nearest-neighbour comparison against fewer than a handful of points is
        not meaningful, so this raises rather than warns.
    duplication_warn_threshold : float, default=0.5
        Emit a ``UserWarning`` when this fraction of synthetic points coincide
        with a real point.
    random_state : int, Generator, SeedSequence or None, default=42
        Seeds the hold-out split. **Which** points get hidden is the single
        largest driver of the error rate, so varying this varies the result.
        The default of 42 reproduces previously documented numbers; ``None``
        draws fresh entropy and is not reproducible.
    stratify_by : ndarray, optional
        Group labels aligned with ``y``. When given, the majority hold-out
        takes ``hidden_ratio`` within each group, so a hold-out cannot miss a
        cluster entirely. Strata are never inferred automatically.
    n_repeats : int, default=1
        Number of independent hold-out splits. Above 1, the returned details
        carry the per-repeat vector and its dispersion. Repeat streams are
        spawned from a ``SeedSequence`` rather than derived as ``seed + i``,
        which would correlate them.
    reseed_oversampler : bool, default=False
        Give the oversampler a fresh seed on each repeat. This changes what the
        dispersion covers -- see Notes.

    Returns
    -------
    float or ValidationDetails
        Error rate by default, ``nan`` if no synthetic samples were produced.
        With ``n_repeats > 1`` the bare return is the mean across repeats.

    Raises
    ------
    ValueError
        If the labels are not binary, if ``minority_label`` is absent, if the
        held-out minority would be smaller than ``min_hidden``, or if the
        oversampler does not preserve the original samples as a prefix.

    Notes
    -----
    The error rate is a **relative** quantity. Its scale depends on
    ``hidden_ratio``, on the density of the data, and on dimensionality, so
    values are not comparable across datasets. See :doc:`/concepts`.

    **What the repeat interval covers.** With ``n_repeats > 1`` the reported
    interval is a percentile bootstrap over the per-repeat error rates. It
    describes the variability of the **hold-out split**, conditional on this
    dataset and on the oversampler's own seed. It is *not* a confidence
    interval for a population quantity, and with ``reseed_oversampler=False``
    it does not include the oversampler's own randomness at all. Setting
    ``reseed_oversampler=True`` clones the sampler with a fresh seed per
    repeat, so the dispersion then covers both sources together -- a different,
    wider decomposition.

    Synthetic points generated from shared parent points are **not
    independent**, so a binomial interval on the error rate would be too
    narrow. Nothing here claims more than the repeat-level bootstrap supports.
    """
    _validate_hidden_ratio(hidden_ratio)
    require_pointwise_metric(metric)
    labels = np.unique(y)
    if minority_label not in labels:
        raise ValueError(f"minority_label {minority_label} not found in y")
    if len(labels) != 2:
        raise ValueError(
            "validate_oversampling expects binary labels; use validate_multiclass_oversampling for multi-class data"
        )
    majority_label = int(labels[labels != minority_label][0])

    if reference not in ("hidden_minority", "train_minority"):
        raise ValueError(
            f"reference must be 'hidden_minority' or 'train_minority'; got {reference!r}"
        )
    warn_reference_bias(reference, stacklevel=3)

    if n_repeats < 1:
        raise ValueError(f"n_repeats must be at least 1; got {n_repeats}")

    if n_repeats > 1:
        return _validate_repeated(
            X,
            y,
            minority_label,
            oversampler,
            hidden_ratio=hidden_ratio,
            metric=metric,
            metric_kwargs=metric_kwargs,
            return_details=return_details,
            reference=reference,
            minority_hidden_ratio=minority_hidden_ratio,
            min_hidden=min_hidden,
            duplication_warn_threshold=duplication_warn_threshold,
            random_state=random_state,
            stratify_by=stratify_by,
            n_repeats=n_repeats,
            reseed_oversampler=reseed_oversampler,
        )

    minority, _ = _split_classes(X, y, minority_label)
    split = prepare_validation_split(
        X,
        y,
        minority_label,
        majority_label,
        hidden_ratio,
        reference=reference,
        minority_hidden_ratio=minority_hidden_ratio,
        min_hidden=min_hidden,
        random_state=random_state,
        stratify_by=stratify_by,
    )
    X_train = split.X_train
    y_train = split.y_train
    hid_majority = split.hid_majority
    fit_minority = split.fit_minority
    reference_minority = split.reference_minority

    try:
        X_res, y_res = oversampler.fit_resample(X_train, y_train)
    except Exception as exc:
        logger.exception("Oversampler failed during fit_resample")
        raise ValueError(
            f"{type(oversampler).__name__} failed to fit on the reduced training "
            f"set ({len(fit_minority)} minority points after holding out "
            f"{len(minority) - len(fit_minority)}). Neighbour-based samplers "
            "such as SMOTE require more minority points than their k_neighbors "
            "setting. Lower k_neighbors, lower minority_hidden_ratio, or supply "
            f"more minority data. Original error: {exc}"
        ) from exc

    synthetic = extract_synthetic_samples(X_train, X_res, y_res, minority_label)

    kwargs = metric_kwargs or {}
    empty = np.empty((0, 0))

    if len(synthetic) == 0:
        warnings.warn(
            f"{type(oversampler).__name__} produced no synthetic minority "
            "samples, so there is nothing to validate. Returning nan rather "
            "than 0.0, which would be indistinguishable from a perfect score.",
            UserWarning,
            stacklevel=2,
        )
        rate = float("nan")
        if return_details:
            return ValidationDetails(
                error_rate=rate,
                n_errors=0,
                n_synthetic=0,
                n_ties=0,
                duplication_rate=float("nan"),
                reference=reference,
                dist_hidden=empty,
                dist_min=empty,
            )
        return rate

    dup_rate = duplication_rate(synthetic, fit_minority)
    if dup_rate >= duplication_warn_threshold:
        warnings.warn(
            f"{dup_rate:.0%} of the synthetic samples produced by "
            f"{type(oversampler).__name__} are exact copies of real minority "
            "points. The validation error rate is not informative for a sampler "
            "that mostly duplicates: copied points sit at distance zero from the "
            "minority set and so can never be scored as errors.",
            UserWarning,
            stacklevel=2,
        )

    dist_hidden = distance_matrix(synthetic, hid_majority, metric, **kwargs)
    dist_min = distance_matrix(synthetic, reference_minority, metric, **kwargs)

    nearest_hidden = dist_hidden.min(axis=1)
    nearest_min = dist_min.min(axis=1)

    errors, n_ties = score_nearest_distances(nearest_hidden, nearest_min)
    rate = calculate_error_rate(errors, len(synthetic))

    if n_ties > 0.01 * len(synthetic):
        warnings.warn(
            f"{n_ties} of {len(synthetic)} synthetic points are exactly "
            "equidistant from the hidden majority and the minority reference "
            "set. Ties are excluded from the error count, but this many "
            "suggests duplicated or heavily quantised features.",
            UserWarning,
            stacklevel=2,
        )

    if return_details:
        return ValidationDetails(
            error_rate=rate,
            n_errors=errors,
            n_synthetic=len(synthetic),
            n_ties=n_ties,
            duplication_rate=dup_rate,
            reference=reference,
            dist_hidden=dist_hidden,
            dist_min=dist_min,
        )

    return rate