MatlabDataset

MatlabDataset loads and saves data to MATLAB files.

kedro_datasets.matlab.MatlabDataset

MatlabDataset(
    filepath,
    save_args=None,
    version=None,
    credentials=None,
    fs_args=None,
    metadata=None,
)

Bases: AbstractVersionedDataset[ndarray, ndarray]

MatlabDataSet loads and saves data from/to a MATLAB file using scipy.io.

Warning

MAT v5 files can contain pickled Python objects. scipy.io.loadmat() will deserialize these, which can execute arbitrary code if the file is untrusted. Only load .mat files from sources you trust.

Examples:

Using the YAML API:

cars:
  type: matlab.MatlabDataset
  filepath: gcs://your_bucket/cars.mat
  fs_args:
  project: my-project
  credentials: my_gcp_credentials

Using the Python API:

>>> import numpy as np
>>> from kedro_datasets.matlab import MatlabDataset
>>>
>>> data = np.array([1, 2, 3])
>>>
>>> dataset = MatlabDataset(filepath=tmp_path / "test.mat")
>>> dataset.save(data)
>>> reloaded = dataset.load()
>>> assert (data == reloaded["data"]).all()

Parameters:

• filepath (str) –

  Filepath in POSIX format to a Matlab file prefixed with a protocol like s3://. If prefix is not provided, file protocol (local filesystem) will be used. The prefix should be any protocol supported by fsspec. Note: http(s) doesn't support versioning.

• save_args (dict[str, Any] | None, default: None ) –

  .mat options for saving .mat files.

• version (Version | None, default: None ) –

  If specified, should be an instance of kedro.io.core.Version. If its load attribute is None, the latest version will be loaded. If its save attribute is None, save version will be autogenerated.

• credentials (dict[str, Any] | None, default: None ) –

  Credentials required to get access to the underlying filesystem. E.g. for GCSFileSystem it should look like {"token": None}.

• fs_args (dict[str, Any] | None, default: None ) –

  Extra arguments to pass into underlying filesystem class constructor (e.g. {"project": "my-project"} for GCSFileSystem), as well as to pass to the filesystem's open method through nested keys open_args_load and open_args_save. Here you can find all available arguments for open: https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.open All defaults are preserved, except mode, which is set to wb when saving.

• metadata (dict[str, Any] | None, default: None ) –

  Any arbitrary metadata. This is ignored by Kedro, but may be consumed by users or external plugins.

Source code in kedro_datasets/matlab/matlab_dataset.py

def __init__(  # noqa = PLR0913
    self,
    filepath: str,
    save_args: dict[str, Any] | None = None,
    version: Version | None = None,
    credentials: dict[str, Any] | None = None,
    fs_args: dict[str, Any] | None = None,
    metadata: dict[str, Any] | None = None,
) -> None:
    """Creates a new instance of MatlabDataSet to load and save data from/to a MATLAB file.

    Args:
        filepath: Filepath in POSIX format to a Matlab file prefixed with a protocol like `s3://`.
            If prefix is not provided, `file` protocol (local filesystem) will be used.
            The prefix should be any protocol supported by ``fsspec``.
            Note: `http(s)` doesn't support versioning.
        save_args: .mat options for saving .mat files.
        version: If specified, should be an instance of
            ``kedro.io.core.Version``. If its ``load`` attribute is
            None, the latest version will be loaded. If its ``save``
            attribute is None, save version will be autogenerated.
        credentials: Credentials required to get access to the underlying filesystem.
            E.g. for ``GCSFileSystem`` it should look like `{"token": None}`.
        fs_args: Extra arguments to pass into underlying filesystem class constructor
            (e.g. `{"project": "my-project"}` for ``GCSFileSystem``), as well as
            to pass to the filesystem's `open` method through nested keys
            `open_args_load` and `open_args_save`.
            Here you can find all available arguments for `open`:
            https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.open
            All defaults are preserved, except `mode`, which is set to `wb` when saving.
        metadata: Any arbitrary metadata.
            This is ignored by Kedro, but may be consumed by users or external plugins.
    """
    _fs_args = deepcopy(fs_args) or {}
    _fs_open_args_load = _fs_args.pop("open_args_load", {})
    _fs_open_args_save = _fs_args.pop("open_args_save", {})
    _credentials = deepcopy(credentials) or {}

    protocol, path = get_protocol_and_path(filepath, version)
    self._protocol = protocol
    if protocol == "file":
        _fs_args.setdefault("auto_mkdir", True)
    self._fs = fsspec.filesystem(self._protocol, **_credentials, **_fs_args)
    self.metadata = metadata

    super().__init__(
        filepath=PurePosixPath(path),
        version=version,
        exists_function=self._fs.exists,
        glob_function=self._fs.glob,
    )
    # Handle default save and fs arguments
    self._save_args = {**self.DEFAULT_SAVE_ARGS, **(save_args or {})}
    self._fs_open_args_load = {
        **self.DEFAULT_FS_ARGS.get("open_args_load", {}),
        **(_fs_open_args_load or {}),
    }
    self._fs_open_args_save = {
        **self.DEFAULT_FS_ARGS.get("open_args_save", {}),
        **(_fs_open_args_save or {}),
    }

DEFAULT_FS_ARGS class-attribute instance-attribute

DEFAULT_FS_ARGS = {'open_args_save': {'mode': 'wb'}}

DEFAULT_SAVE_ARGS class-attribute instance-attribute

DEFAULT_SAVE_ARGS = {'indent': 2}

_fs instance-attribute

_fs = fsspec.filesystem(
    self._protocol, **_credentials, **_fs_args
)

_fs_open_args_load instance-attribute

_fs_open_args_load = {
    None: self.DEFAULT_FS_ARGS.get("open_args_load", {}),
    None: _fs_open_args_load or {},
}

_fs_open_args_save instance-attribute

_fs_open_args_save = {
    None: self.DEFAULT_FS_ARGS.get("open_args_save", {}),
    None: _fs_open_args_save or {},
}

_protocol instance-attribute

_protocol = protocol

_save_args instance-attribute

_save_args = {
    None: self.DEFAULT_SAVE_ARGS,
    None: save_args or {},
}

metadata instance-attribute

metadata = metadata

_describe

_describe()

Source code in kedro_datasets/matlab/matlab_dataset.py

def _describe(self) -> dict[str, Any]:
    return {
        "filepath": self._filepath,
        "protocol": self._protocol,
        "save_args": self._save_args,
        "version": self._version,
    }

_exists

_exists()

Source code in kedro_datasets/matlab/matlab_dataset.py

def _exists(self) -> bool:
    try:
        load_path = get_filepath_str(self._get_load_path(), self._protocol)
    except DatasetError:
        return False

    return self._fs.exists(load_path)

_invalidate_cache

_invalidate_cache()

Invalidate underlying filesystem caches.

Source code in kedro_datasets/matlab/matlab_dataset.py

def _invalidate_cache(self) -> None:
    """Invalidate underlying filesystem caches."""
    filepath = get_filepath_str(self._filepath, self._protocol)
    self._fs.invalidate_cache(filepath)

_release

_release()

Source code in kedro_datasets/matlab/matlab_dataset.py

def _release(self) -> None:
    super()._release()
    self._invalidate_cache()

load

load()

Access the specific variable in the .mat file, e.g, data['variable_name']

Source code in kedro_datasets/matlab/matlab_dataset.py

def load(self) -> np.ndarray:
    """
    Access the specific variable in the .mat file, e.g, data['variable_name']
    """
    load_path = get_filepath_str(self._get_load_path(), self._protocol)
    with self._fs.open(load_path) as f:
        data = io.loadmat(f)
        return data

save

save(data)

Source code in kedro_datasets/matlab/matlab_dataset.py

def save(self, data: np.ndarray) -> None:
    save_path = get_filepath_str(self._get_save_path(), self._protocol)
    with self._fs.open(save_path, **self._fs_open_args_save) as f:
        io.savemat(f, {"data": data})
    self._invalidate_cache()