Skip to content

XMLDataset

XMLDataset loads and saves data to/from XML files using pandas.

kedro_datasets.pandas.XMLDataset 

XMLDataset(
    *,
    filepath,
    load_args=None,
    save_args=None,
    version=None,
    credentials=None,
    fs_args=None,
    metadata=None
)

Bases: AbstractVersionedDataset[DataFrame, DataFrame]

XMLDataset loads/saves data from/to a XML file using an underlying filesystem (e.g.: local, S3, GCS). It uses pandas to handle the XML file.

Examples:

Using the Python API:

>>> import pandas as pd
>>> from kedro_datasets.pandas import XMLDataset
>>>
>>> data = pd.DataFrame({"col1": [1, 2], "col2": [4, 5], "col3": [5, 6]})
>>>
>>> dataset = XMLDataset(filepath=tmp_path / "test.xml")
>>> dataset.save(data)
>>> reloaded = dataset.load()
>>> assert data.equals(reloaded)

Parameters:

  • filepath (str) –

    Filepath in POSIX format to a XML file prefixed with a protocol like s3://. If prefix is not provided, file protocol (local filesystem) will be used. The prefix should be any protocol supported by fsspec. Note: http(s) doesn't support versioning.

  • load_args (dict[str, Any] | None, default: None ) –

    Pandas options for loading XML files. Here you can find all available arguments: https://pandas.pydata.org/docs/reference/api/pandas.read_xml.html All defaults are preserved.

  • save_args (dict[str, Any] | None, default: None ) –

    Pandas options for saving XML files. Here you can find all available arguments: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_xml.html All defaults are preserved, but "index", which is set to False.

  • version (Version | None, default: None ) –

    If specified, should be an instance of kedro.io.core.Version. If its load attribute is None, the latest version will be loaded. If its save attribute is None, save version will be autogenerated.

  • credentials (dict[str, Any] | None, default: None ) –

    Credentials required to get access to the underlying filesystem. E.g. for GCSFileSystem it should look like {"token": None}.

  • fs_args (dict[str, Any] | None, default: None ) –

    Extra arguments to pass into underlying filesystem class constructor (e.g. {"project": "my-project"} for GCSFileSystem). Defaults are preserved, apart from the open_args_save mode which is set to wb. Note that the save method requires bytes, so any save mode provided should include "b" for bytes.

  • metadata (dict[str, Any] | None, default: None ) –

    Any arbitrary metadata. This is ignored by Kedro, but may be consumed by users or external plugins.

Source code in kedro_datasets/pandas/xml_dataset.py 
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
def __init__(  # noqa: PLR0913
    self,
    *,
    filepath: str,
    load_args: dict[str, Any] | None = None,
    save_args: dict[str, Any] | None = None,
    version: Version | None = None,
    credentials: dict[str, Any] | None = None,
    fs_args: dict[str, Any] | None = None,
    metadata: dict[str, Any] | None = None,
) -> None:
    """Creates a new instance of ``XMLDataset`` pointing to a concrete XML file
    on a specific filesystem.

    Args:
        filepath: Filepath in POSIX format to a XML file prefixed with a protocol like `s3://`.
            If prefix is not provided, `file` protocol (local filesystem) will be used.
            The prefix should be any protocol supported by ``fsspec``.
            Note: `http(s)` doesn't support versioning.
        load_args: Pandas options for loading XML files.
            Here you can find all available arguments:
            https://pandas.pydata.org/docs/reference/api/pandas.read_xml.html
            All defaults are preserved.
        save_args: Pandas options for saving XML files.
            Here you can find all available arguments:
            https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_xml.html
            All defaults are preserved, but "index", which is set to False.
        version: If specified, should be an instance of
            ``kedro.io.core.Version``. If its ``load`` attribute is
            None, the latest version will be loaded. If its ``save``
            attribute is None, save version will be autogenerated.
        credentials: Credentials required to get access to the underlying filesystem.
            E.g. for ``GCSFileSystem`` it should look like `{"token": None}`.
        fs_args: Extra arguments to pass into underlying filesystem class constructor
            (e.g. `{"project": "my-project"}` for ``GCSFileSystem``).
            Defaults are preserved, apart from the `open_args_save` `mode` which is set to `wb`.
            Note that the save method requires bytes, so any save mode provided should include "b" for bytes.
        metadata: Any arbitrary metadata.
            This is ignored by Kedro, but may be consumed by users or external plugins.
    """
    _fs_args = deepcopy(fs_args) or {}
    _fs_open_args_load = _fs_args.pop("open_args_load", {})
    _fs_open_args_save = _fs_args.pop("open_args_save", {})
    _credentials = deepcopy(credentials) or {}

    protocol, path = get_protocol_and_path(filepath, version)
    if protocol == "file":
        _fs_args.setdefault("auto_mkdir", True)

    self._protocol = protocol
    self._storage_options = {**_credentials, **_fs_args}
    self._fs = fsspec.filesystem(self._protocol, **self._storage_options)

    self.metadata = metadata

    super().__init__(
        filepath=PurePosixPath(path),
        version=version,
        exists_function=self._fs.exists,
        glob_function=self._fs.glob,
    )

    # Handle default load and save and fs arguments
    self._load_args = {**self.DEFAULT_LOAD_ARGS, **(load_args or {})}
    self._save_args = {**self.DEFAULT_SAVE_ARGS, **(save_args or {})}
    self._fs_open_args_load = {
        **self.DEFAULT_FS_ARGS.get("open_args_load", {}),
        **(_fs_open_args_load or {}),
    }
    self._fs_open_args_save = {
        **self.DEFAULT_FS_ARGS.get("open_args_save", {}),
        **(_fs_open_args_save or {}),
    }

    if "storage_options" in self._save_args or "storage_options" in self._load_args:
        logger.warning(
            "Dropping 'storage_options' for %s, "
            "please specify them under 'fs_args' or 'credentials'.",
            self._filepath,
        )
        self._save_args.pop("storage_options", None)
        self._load_args.pop("storage_options", None)

DEFAULT_FS_ARGS class-attribute instance-attribute 

DEFAULT_FS_ARGS = {'open_args_save': {'mode': 'wb'}}

DEFAULT_LOAD_ARGS class-attribute instance-attribute 

DEFAULT_LOAD_ARGS = {}

DEFAULT_SAVE_ARGS class-attribute instance-attribute 

DEFAULT_SAVE_ARGS = {'index': False}

_fs instance-attribute 

_fs = filesystem(_protocol, **(_storage_options))

_fs_open_args_load instance-attribute 

_fs_open_args_load = {
    None: get("open_args_load", {}),
    None: _fs_open_args_load or {},
}

_fs_open_args_save instance-attribute 

_fs_open_args_save = {
    None: get("open_args_save", {}),
    None: _fs_open_args_save or {},
}

_load_args instance-attribute 

_load_args = {
    None: DEFAULT_LOAD_ARGS,
    None: load_args or {},
}

_protocol instance-attribute 

_protocol = protocol

_save_args instance-attribute 

_save_args = {
    None: DEFAULT_SAVE_ARGS,
    None: save_args or {},
}

_storage_options instance-attribute 

_storage_options = {None: _credentials, None: _fs_args}

metadata instance-attribute 

metadata = metadata

_describe 

_describe()
Source code in kedro_datasets/pandas/xml_dataset.py 
131
132
133
134
135
136
137
138
def _describe(self) -> dict[str, Any]:
    return {
        "filepath": self._filepath,
        "protocol": self._protocol,
        "load_args": self._load_args,
        "save_args": self._save_args,
        "version": self._version,
    }

_exists 

_exists()
Source code in kedro_datasets/pandas/xml_dataset.py 
162
163
164
165
166
167
168
def _exists(self) -> bool:
    try:
        load_path = get_filepath_str(self._get_load_path(), self._protocol)
    except DatasetError:
        return False

    return self._fs.exists(load_path)

_invalidate_cache 

_invalidate_cache()

Invalidate underlying filesystem caches.

Source code in kedro_datasets/pandas/xml_dataset.py 
174
175
176
177
def _invalidate_cache(self) -> None:
    """Invalidate underlying filesystem caches."""
    filepath = get_filepath_str(self._filepath, self._protocol)
    self._fs.invalidate_cache(filepath)

_release 

_release()
Source code in kedro_datasets/pandas/xml_dataset.py 
170
171
172
def _release(self) -> None:
    super()._release()
    self._invalidate_cache()

load 

load()
Source code in kedro_datasets/pandas/xml_dataset.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
def load(self) -> pd.DataFrame:
    load_path = str(self._get_load_path())
    if self._protocol == "file":
        # file:// protocol seems to misbehave on Windows
        # (<urlopen error file not on local host>),
        # so we don't join that back to the filepath;
        # storage_options also don't work with local paths
        return pd.read_xml(load_path, **self._load_args)

    load_path = f"{self._protocol}{PROTOCOL_DELIMITER}{load_path}"
    return pd.read_xml(
        load_path, storage_options=self._storage_options, **self._load_args
    )

save 

save(data)
Source code in kedro_datasets/pandas/xml_dataset.py 
154
155
156
157
158
159
160
def save(self, data: pd.DataFrame) -> None:
    save_path = get_filepath_str(self._get_save_path(), self._protocol)

    with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
        data.to_xml(path_or_buffer=fs_file, **self._save_args)

    self._invalidate_cache()
Copyright © Kedro, a Series of LF Projects, LLC