Skip to content

oversampleqa.reports

oversampleqa.reports

One report object, and the metadata that makes it auditable.

Three surfaces had drifted apart: validate_oversampling returned a float or a tuple, the inference layer returned its own dataclasses, and the fidelity suite returned a third set. Every consumer -- CLI, reporting, plotting, benchmarks -- needed bespoke handling for each. :class:ValidationReport composes them, so a consumer handles one shape.

Exported results outlive the code that produced them, so every export carries a schema_version and the metadata needed to reproduce the run.

SCHEMA_VERSION = '1.0' module-attribute

Version of the exported JSON structure.

Bump the minor part for additive changes and the major part when a field is removed or changes meaning. Consumers should refuse a major version they do not recognise rather than guess.

RunMetadata dataclass

Everything needed to reproduce and audit a run.

A number without its provenance is not a result. This records the package and dependency versions, the sampler and its parameters, the seed, and a hash of the data -- so a report exported today can be checked against a rerun in a year, and a mismatch localised to whichever of those changed.

Source code in src/oversampleqa/reports.py 
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
@dataclass(frozen=True)
class RunMetadata:
    """Everything needed to reproduce and audit a run.

    A number without its provenance is not a result. This records the package
    and dependency versions, the sampler and its parameters, the seed, and a
    hash of the data -- so a report exported today can be checked against a
    rerun in a year, and a mismatch localised to whichever of those changed.
    """

    oversampler: str = ""
    oversampler_params: dict[str, Any] = field(default_factory=dict)
    metric: str = "hassanat"
    hidden_ratio: float = 0.1
    reference: str = "hidden_minority"
    random_state: int | None = None
    n_repeats: int = 1
    dataset: str = ""
    dataset_hash: str = ""
    n_samples: int = 0
    n_features: int = 0
    minority_label: int | None = None
    oversampleqa_version: str = ""
    numpy_version: str = ""
    sklearn_version: str = ""
    imblearn_version: str = ""
    timestamp: str = ""

    @classmethod
    def capture(
        cls,
        X: NDArray[np.floating],
        y: NDArray[np.integer],
        oversampler: Any,
        *,
        minority_label: int | None = None,
        metric: str = "hassanat",
        hidden_ratio: float = 0.1,
        reference: str = "hidden_minority",
        random_state: int | None = None,
        n_repeats: int = 1,
    ) -> RunMetadata:
        """Collect metadata for a run about to happen, or just completed."""
        import sklearn
        from imblearn import __version__ as imblearn_version

        from . import __version__ as package_version

        params: dict[str, Any] = {}
        if hasattr(oversampler, "get_params"):
            params = {k: repr(v) for k, v in oversampler.get_params().items()}

        X_arr = np.asarray(X)
        return cls(
            oversampler=type(oversampler).__name__,
            oversampler_params=params,
            metric=metric,
            hidden_ratio=hidden_ratio,
            reference=reference,
            random_state=random_state,
            n_repeats=n_repeats,
            dataset_hash=_dataset_hash(X, y),
            n_samples=int(X_arr.shape[0]),
            n_features=int(X_arr.shape[1]) if X_arr.ndim > 1 else 1,
            minority_label=minority_label,
            oversampleqa_version=package_version,
            numpy_version=np.__version__,
            sklearn_version=sklearn.__version__,
            imblearn_version=imblearn_version,
            timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
        )

    def to_dict(self) -> dict[str, Any]:
        """JSON-safe mapping."""
        payload: dict[str, Any] = _json_safe(
            {
                "oversampler": self.oversampler,
                "oversampler_params": self.oversampler_params,
                "metric": self.metric,
                "hidden_ratio": self.hidden_ratio,
                "reference": self.reference,
                "random_state": self.random_state,
                "n_repeats": self.n_repeats,
                "dataset_hash": self.dataset_hash,
                "n_samples": self.n_samples,
                "n_features": self.n_features,
                "minority_label": self.minority_label,
                "oversampleqa_version": self.oversampleqa_version,
                "numpy_version": self.numpy_version,
                "sklearn_version": self.sklearn_version,
                "imblearn_version": self.imblearn_version,
                "timestamp": self.timestamp,
            }
        )
        return payload

    @classmethod
    def from_dict(cls, payload: dict[str, Any]) -> RunMetadata:
        """Rebuild from :meth:`to_dict` output, ignoring unknown keys."""
        known = set(cls.__dataclass_fields__)
        return cls(**{k: v for k, v in payload.items() if k in known})

capture(X, y, oversampler, *, minority_label=None, metric='hassanat', hidden_ratio=0.1, reference='hidden_minority', random_state=None, n_repeats=1) classmethod

Collect metadata for a run about to happen, or just completed.

Source code in src/oversampleqa/reports.py 
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
@classmethod
def capture(
    cls,
    X: NDArray[np.floating],
    y: NDArray[np.integer],
    oversampler: Any,
    *,
    minority_label: int | None = None,
    metric: str = "hassanat",
    hidden_ratio: float = 0.1,
    reference: str = "hidden_minority",
    random_state: int | None = None,
    n_repeats: int = 1,
) -> RunMetadata:
    """Collect metadata for a run about to happen, or just completed."""
    import sklearn
    from imblearn import __version__ as imblearn_version

    from . import __version__ as package_version

    params: dict[str, Any] = {}
    if hasattr(oversampler, "get_params"):
        params = {k: repr(v) for k, v in oversampler.get_params().items()}

    X_arr = np.asarray(X)
    return cls(
        oversampler=type(oversampler).__name__,
        oversampler_params=params,
        metric=metric,
        hidden_ratio=hidden_ratio,
        reference=reference,
        random_state=random_state,
        n_repeats=n_repeats,
        dataset_hash=_dataset_hash(X, y),
        n_samples=int(X_arr.shape[0]),
        n_features=int(X_arr.shape[1]) if X_arr.ndim > 1 else 1,
        minority_label=minority_label,
        oversampleqa_version=package_version,
        numpy_version=np.__version__,
        sklearn_version=sklearn.__version__,
        imblearn_version=imblearn_version,
        timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
    )

to_dict()

JSON-safe mapping.

Source code in src/oversampleqa/reports.py 
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
def to_dict(self) -> dict[str, Any]:
    """JSON-safe mapping."""
    payload: dict[str, Any] = _json_safe(
        {
            "oversampler": self.oversampler,
            "oversampler_params": self.oversampler_params,
            "metric": self.metric,
            "hidden_ratio": self.hidden_ratio,
            "reference": self.reference,
            "random_state": self.random_state,
            "n_repeats": self.n_repeats,
            "dataset_hash": self.dataset_hash,
            "n_samples": self.n_samples,
            "n_features": self.n_features,
            "minority_label": self.minority_label,
            "oversampleqa_version": self.oversampleqa_version,
            "numpy_version": self.numpy_version,
            "sklearn_version": self.sklearn_version,
            "imblearn_version": self.imblearn_version,
            "timestamp": self.timestamp,
        }
    )
    return payload

from_dict(payload) classmethod

Rebuild from :meth:to_dict output, ignoring unknown keys.

Source code in src/oversampleqa/reports.py 
148
149
150
151
152
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> RunMetadata:
    """Rebuild from :meth:`to_dict` output, ignoring unknown keys."""
    known = set(cls.__dataclass_fields__)
    return cls(**{k: v for k, v in payload.items() if k in known})

ValidationReport dataclass

Everything known about one oversampler on one dataset.

calibration, inference and fidelity are optional because each costs real time: the calibration fits nothing but resamples repeatedly, the two-sample tests permute, and the fidelity suite can fit models. A report with only error_rate and details is the cheap default.

Source code in src/oversampleqa/reports.py 
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
@dataclass(frozen=True)
class ValidationReport:
    """Everything known about one oversampler on one dataset.

    ``calibration``, ``inference`` and ``fidelity`` are optional because each
    costs real time: the calibration fits nothing but resamples repeatedly, the
    two-sample tests permute, and the fidelity suite can fit models. A report
    with only ``error_rate`` and ``details`` is the cheap default.
    """

    error_rate: float
    metadata: RunMetadata
    details: Any = None
    calibration: Any = None
    inference: Any = None
    fidelity: Any = None
    schema_version: str = SCHEMA_VERSION

    def to_dict(self) -> dict[str, Any]:
        """JSON-serialisable mapping of the whole report.

        Non-finite floats become ``null``; see :func:`_json_safe`.
        """
        payload: dict[str, Any] = {
            "schema_version": self.schema_version,
            "error_rate": _json_safe(self.error_rate),
            "metadata": self.metadata.to_dict(),
        }
        for name in ("details", "calibration", "inference", "fidelity"):
            component = getattr(self, name)
            if component is None:
                payload[name] = None
            elif hasattr(component, "to_dict"):
                payload[name] = _json_safe(component.to_dict())
            else:  # pragma: no cover - defensive
                payload[name] = _json_safe(component)
        return payload

    @classmethod
    def from_dict(cls, payload: dict[str, Any]) -> ValidationReport:
        """Rebuild from :meth:`to_dict` output.

        Components come back as plain dicts rather than their original
        dataclasses: the export is the interchange format, and rehydrating each
        component type would couple this module to every one of them. Round
        trips are therefore compared on ``to_dict()``, which is what a consumer
        actually reads.
        """
        version = payload.get("schema_version", "0")
        if version.split(".")[0] != SCHEMA_VERSION.split(".")[0]:
            raise ValueError(
                f"report schema version {version} is not compatible with "
                f"{SCHEMA_VERSION}; a major-version change means a field was "
                "removed or changed meaning, so this cannot be read safely"
            )
        return cls(
            error_rate=(
                float("nan")
                if payload.get("error_rate") is None
                else float(payload["error_rate"])
            ),
            metadata=RunMetadata.from_dict(payload.get("metadata", {})),
            details=payload.get("details"),
            calibration=payload.get("calibration"),
            inference=payload.get("inference"),
            fidelity=payload.get("fidelity"),
            schema_version=version,
        )

    def to_json(self, indent: int = 2) -> str:
        """Serialise to JSON. ``allow_nan=False`` guarantees valid output."""
        return strict_json_dumps(self.to_dict(), indent=indent)

    def to_frame(self) -> pd.DataFrame:
        """Tidy one-row frame with every scalar flattened."""
        flat: dict[str, Any] = {
            "schema_version": self.schema_version,
            "error_rate": self.error_rate,
            # `dataset` is whatever the caller named it and is empty when they
            # named nothing: this surface validates arrays, not a file, so it
            # has no name of its own. `dataset_hash` is the identity that is
            # always present, and is promoted out of the `meta_` block because
            # a row nobody can trace back to its data is not much of a record.
            "dataset": self.metadata.dataset,
            "dataset_hash": self.metadata.dataset_hash,
            "oversampler": self.metadata.oversampler,
            "metric": self.metadata.metric,
            "hidden_ratio": self.metadata.hidden_ratio,
            "reference": self.metadata.reference,
            "random_state": self.metadata.random_state,
            "n_repeats": self.metadata.n_repeats,
            "minority_label": self.metadata.minority_label,
            "oversampleqa_version": self.metadata.oversampleqa_version,
        }
        flat.update(
            {
                f"meta_{k}": v
                for k, v in self.metadata.to_dict().items()
                if not isinstance(v, (dict, list))
            }
        )
        for name in ("details", "calibration", "inference", "fidelity"):
            component = getattr(self, name)
            if component is None or not hasattr(component, "to_dict"):
                continue
            for key, value in component.to_dict().items():
                if not isinstance(value, (dict, list, tuple, np.ndarray)):
                    flat[f"{name}_{key}"] = value
        return pd.DataFrame([flat])

    def __rich__(self) -> str:
        """Compact CLI rendering."""
        lines = [
            f"[bold]OversampleQA report[/bold] (schema {self.schema_version})",
            f"  error rate     {self.error_rate:.4f}",
            f"  oversampler    {self.metadata.oversampler}",
            f"  metric         {self.metadata.metric}",
            f"  random_state   {self.metadata.random_state}",
            f"  dataset        {self.metadata.dataset_hash} "
            f"({self.metadata.n_samples}x{self.metadata.n_features})",
        ]
        if self.calibration is not None and hasattr(self.calibration, "interpret"):
            lines.append(f"  calibration    {self.calibration.interpret()}")
        if self.fidelity is not None and hasattr(self.fidelity, "interpret"):
            lines.extend(f"  fidelity       {n}" for n in self.fidelity.interpret())
        return "\n".join(lines)

    def with_components(self, **components: Any) -> ValidationReport:
        """Return a copy carrying additional components."""
        return replace(self, **components)

to_dict()

JSON-serialisable mapping of the whole report.

Non-finite floats become null; see :func:_json_safe.

Source code in src/oversampleqa/reports.py 
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def to_dict(self) -> dict[str, Any]:
    """JSON-serialisable mapping of the whole report.

    Non-finite floats become ``null``; see :func:`_json_safe`.
    """
    payload: dict[str, Any] = {
        "schema_version": self.schema_version,
        "error_rate": _json_safe(self.error_rate),
        "metadata": self.metadata.to_dict(),
    }
    for name in ("details", "calibration", "inference", "fidelity"):
        component = getattr(self, name)
        if component is None:
            payload[name] = None
        elif hasattr(component, "to_dict"):
            payload[name] = _json_safe(component.to_dict())
        else:  # pragma: no cover - defensive
            payload[name] = _json_safe(component)
    return payload

from_dict(payload) classmethod

Rebuild from :meth:to_dict output.

Components come back as plain dicts rather than their original dataclasses: the export is the interchange format, and rehydrating each component type would couple this module to every one of them. Round trips are therefore compared on to_dict(), which is what a consumer actually reads.

Source code in src/oversampleqa/reports.py 
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> ValidationReport:
    """Rebuild from :meth:`to_dict` output.

    Components come back as plain dicts rather than their original
    dataclasses: the export is the interchange format, and rehydrating each
    component type would couple this module to every one of them. Round
    trips are therefore compared on ``to_dict()``, which is what a consumer
    actually reads.
    """
    version = payload.get("schema_version", "0")
    if version.split(".")[0] != SCHEMA_VERSION.split(".")[0]:
        raise ValueError(
            f"report schema version {version} is not compatible with "
            f"{SCHEMA_VERSION}; a major-version change means a field was "
            "removed or changed meaning, so this cannot be read safely"
        )
    return cls(
        error_rate=(
            float("nan")
            if payload.get("error_rate") is None
            else float(payload["error_rate"])
        ),
        metadata=RunMetadata.from_dict(payload.get("metadata", {})),
        details=payload.get("details"),
        calibration=payload.get("calibration"),
        inference=payload.get("inference"),
        fidelity=payload.get("fidelity"),
        schema_version=version,
    )

to_json(indent=2)

Serialise to JSON. allow_nan=False guarantees valid output.

Source code in src/oversampleqa/reports.py 
224
225
226
def to_json(self, indent: int = 2) -> str:
    """Serialise to JSON. ``allow_nan=False`` guarantees valid output."""
    return strict_json_dumps(self.to_dict(), indent=indent)

to_frame()

Tidy one-row frame with every scalar flattened.

Source code in src/oversampleqa/reports.py 
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
def to_frame(self) -> pd.DataFrame:
    """Tidy one-row frame with every scalar flattened."""
    flat: dict[str, Any] = {
        "schema_version": self.schema_version,
        "error_rate": self.error_rate,
        # `dataset` is whatever the caller named it and is empty when they
        # named nothing: this surface validates arrays, not a file, so it
        # has no name of its own. `dataset_hash` is the identity that is
        # always present, and is promoted out of the `meta_` block because
        # a row nobody can trace back to its data is not much of a record.
        "dataset": self.metadata.dataset,
        "dataset_hash": self.metadata.dataset_hash,
        "oversampler": self.metadata.oversampler,
        "metric": self.metadata.metric,
        "hidden_ratio": self.metadata.hidden_ratio,
        "reference": self.metadata.reference,
        "random_state": self.metadata.random_state,
        "n_repeats": self.metadata.n_repeats,
        "minority_label": self.metadata.minority_label,
        "oversampleqa_version": self.metadata.oversampleqa_version,
    }
    flat.update(
        {
            f"meta_{k}": v
            for k, v in self.metadata.to_dict().items()
            if not isinstance(v, (dict, list))
        }
    )
    for name in ("details", "calibration", "inference", "fidelity"):
        component = getattr(self, name)
        if component is None or not hasattr(component, "to_dict"):
            continue
        for key, value in component.to_dict().items():
            if not isinstance(value, (dict, list, tuple, np.ndarray)):
                flat[f"{name}_{key}"] = value
    return pd.DataFrame([flat])

__rich__()

Compact CLI rendering.

Source code in src/oversampleqa/reports.py 
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
def __rich__(self) -> str:
    """Compact CLI rendering."""
    lines = [
        f"[bold]OversampleQA report[/bold] (schema {self.schema_version})",
        f"  error rate     {self.error_rate:.4f}",
        f"  oversampler    {self.metadata.oversampler}",
        f"  metric         {self.metadata.metric}",
        f"  random_state   {self.metadata.random_state}",
        f"  dataset        {self.metadata.dataset_hash} "
        f"({self.metadata.n_samples}x{self.metadata.n_features})",
    ]
    if self.calibration is not None and hasattr(self.calibration, "interpret"):
        lines.append(f"  calibration    {self.calibration.interpret()}")
    if self.fidelity is not None and hasattr(self.fidelity, "interpret"):
        lines.extend(f"  fidelity       {n}" for n in self.fidelity.interpret())
    return "\n".join(lines)

with_components(**components)

Return a copy carrying additional components.

Source code in src/oversampleqa/reports.py 
282
283
284
def with_components(self, **components: Any) -> ValidationReport:
    """Return a copy carrying additional components."""
    return replace(self, **components)