Skip to content

Loader

Dataset → DataQuality extraction (format-agnostic).

File parsing is delegated to :class:src.data_loader.DataLoader; this module computes the SimpleMetaFeatures-equivalent counts plus the pymfe meta-features.

load_qualities(dataset, *, data_format='arff', did=None, evaluation_engine_id=None, groups=_DEFAULT_MFE_GROUPS, random_state=42, timeout=30)

Extract a :class:DataQuality from a downloaded dataset.

Parameters

dataset: Already-downloaded dataset (file_path points at an .arff or .parquet file depending on how it was fetched). data_format: "arff" (default) or "parquet" — selects the :class:~src.data_loader.DataLoader backend.

Source code in src/qualities/loader.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
def load_qualities(
    dataset: DatasetDownloadInfo,
    *,
    data_format: DataFormat = "arff",
    did: int | None = None,
    evaluation_engine_id: int | None = None,
    groups: tuple[str, ...] = _DEFAULT_MFE_GROUPS,
    random_state: int = 42,
    timeout: int = 30,
) -> DataQuality:
    """Extract a :class:`DataQuality` from a downloaded dataset.

    Parameters
    ----------
    dataset:
        Already-downloaded dataset (``file_path`` points at an ``.arff`` or
        ``.parquet`` file depending on how it was fetched).
    data_format:
        ``"arff"`` (default) or ``"parquet"`` — selects the
        :class:`~src.data_loader.DataLoader` backend.
    """
    try:
        attributes, rows = DataLoader(data_format).load(dataset)

        target_names = normalize_target_names(
            dataset.default_target_attribute,
        )

        qualities = compute_dataset_qualities(
            attributes,
            rows,
            target_names,
        )

        X, y = build_xy(attributes, rows, target_names)

        mfe = MFE(
            groups=tuple(groups),
            random_state=random_state,
        )
        mfe.fit(X, y)
        names, values = mfe.extract(
            cat_cols="auto",
            suppress_warnings=True,
            verbose=0,
            timeout=timeout,
        )

        for name, value in zip(names, values):
            if value is None:
                parsed: float | None = None
            else:
                try:
                    parsed = float(value)
                except (TypeError, ValueError):
                    parsed = None

            if parsed is not None and math.isnan(parsed):
                parsed = None

            qualities.append(Quality(name=str(name), value=parsed))

    except Exception as exc:
        return DataQuality(
            did=did,
            evaluation_engine_id=evaluation_engine_id,
            error=str(exc),
        )

    return DataQuality(
        did=did,
        evaluation_engine_id=evaluation_engine_id,
        qualities=qualities,
    )