Skip to content

Loader

Dataset → DataQuality extraction (format-agnostic).

File parsing is delegated to :class:src.data_loader.DataLoader; this module computes the SimpleMetaFeatures-equivalent counts plus the pymfe meta-features.

load_qualities(dataset, *, data_format='arff', did=None, evaluation_engine_id=None, groups=_DEFAULT_MFE_GROUPS, random_state=42, timeout=30)

Extract a :class:DataQuality from a downloaded dataset.

Parameters

dataset: Already-downloaded dataset (file_path points at an .arff or .parquet file depending on how it was fetched). data_format: "arff" (default) or "parquet" — selects the :class:~src.data_loader.DataLoader backend.

Source code in src/qualities/loader.py
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
def load_qualities(
    dataset: DatasetDownloadInfo,
    *,
    data_format: DataFormat = "arff",
    did: int | None = None,
    evaluation_engine_id: int | None = None,
    groups: tuple[str, ...] = _DEFAULT_MFE_GROUPS,
    random_state: int = 42,
    timeout: int = 30,
) -> DataQuality:
    """Extract a :class:`DataQuality` from a downloaded dataset.

    Parameters
    ----------
    dataset:
        Already-downloaded dataset (``file_path`` points at an ``.arff`` or
        ``.parquet`` file depending on how it was fetched).
    data_format:
        ``"arff"`` (default) or ``"parquet"`` — selects the
        :class:`~src.data_loader.DataLoader` backend.
    """
    try:
        attributes, rows = DataLoader(data_format).load(dataset)

        target_names = normalize_target_names(
            dataset.default_target_attribute,
        )

        qualities = compute_dataset_qualities(
            attributes,
            rows,
            target_names,
        )

        # Supervised extraction (pymfe, and landmarkers upstream) needs a
        # target column. Without one, skip it explicitly — the
        # target-dependent qualities are already None in ``qualities`` —
        # rather than letting build_xy raise into the except below.
        if find_target_index(attributes, target_names) is None:
            return DataQuality(
                did=did,
                evaluation_engine_id=evaluation_engine_id,
                qualities=qualities,
            )

        X, y = build_xy(attributes, rows, target_names)

        mfe = MFE(
            groups=tuple(groups),
            random_state=random_state,
        )
        mfe.fit(X, y)
        names, values = mfe.extract(
            cat_cols="auto",
            suppress_warnings=True,
            verbose=0,
            timeout=timeout,
        )

        for name, value in zip(names, values):
            if value is None:
                parsed: float | None = None
            else:
                try:
                    parsed = float(value)
                except (TypeError, ValueError):
                    parsed = None

            if parsed is not None and math.isnan(parsed):
                parsed = None

            qualities.append(Quality(name=str(name), value=parsed))

    except Exception as exc:
        return DataQuality(
            did=did,
            evaluation_engine_id=evaluation_engine_id,
            error=str(exc),
        )

    return DataQuality(
        did=did,
        evaluation_engine_id=evaluation_engine_id,
        qualities=qualities,
    )