Skip to content

permission error xarray

Bug

Error description

Downloaded data are not always possible to store as xarray due to a PermissionError

Error message

2022-07-07 22:04:10,125 - ERROR: [Errno 13] Permission denied: b'/p/project/deepacf/intelliaq/leufen1/DATA/MLAIR/daily/AT0ZOE2_o3.nc' [run_environment.py:__exit__:137]

  File "/p/home/jusers/leufen1/juwels/intelliaq/mlair_tf2/mlair/mlair/data_handler/data_handler_single_station.py", line 269, in setup_samples
    self.load_lazy()
  File "/p/home/jusers/leufen1/juwels/intelliaq/mlair_tf2/mlair/mlair/data_handler/data_handler_single_station.py", line 297, in load_lazy
    self.make_input_target()
  File "/p/home/jusers/leufen1/juwels/intelliaq/mlair_tf2/mlair/mlair/data_handler/data_handler_mixed_sampling.py", line 121, in make_input_target
    self._data = tuple(map(self.load_and_interpolate, [0, 1]))  # load input (0) and target (1) data
  File "/p/home/jusers/leufen1/juwels/intelliaq/mlair_tf2/mlair/mlair/data_handler/data_handler_mixed_sampling.py", line 149, in load_and_interpolate
    data, self.meta = self.load_data(self.path[ind], self.station, stats_per_var, self.sampling[ind],
  File "/p/home/jusers/leufen1/juwels/intelliaq/mlair_tf2/mlair/mlair/data_handler/data_handler_single_station.py", line 358, in load_data
    data, meta = self.download_data(file_name, meta_file, station, statistics_per_var, sampling,
  File "/p/home/jusers/leufen1/juwels/intelliaq/mlair_tf2/mlair/mlair/data_handler/data_handler_single_station.py", line 423, in download_data
    xarr.to_netcdf(path=file_name, format="NETCDF4", engine="netcdf4", mode="w")
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/core/dataarray.py", line 2664, in to_netcdf
    return dataset.to_netcdf(*args, **kwargs)
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/core/dataset.py", line 1644, in to_netcdf
    return to_netcdf(
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/backends/api.py", line 1094, in to_netcdf
    store = store_open(target, mode, format, group, **kwargs)
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/backends/netCDF4_.py", line 364, in open
    return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/backends/netCDF4_.py", line 314, in __init__
    self.format = self.ds.data_model
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/backends/netCDF4_.py", line 373, in ds
    return self._acquire()
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/backends/netCDF4_.py", line 367, in _acquire
    with self._manager.acquire_context(needs_lock) as root:
  File "/p/software/hdfml/stages/2020/software/Python/3.8.5-GCCcore-10.3.0/lib/python3.8/contextlib.py", line 113, in __enter__
    return next(self.gen)
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/backends/file_manager.py", line 187, in acquire_context
    file, cached = self._acquire_with_cache_info(needs_lock)
  File "/p/software/hdfml/stages/2020/software/Jupyter/2021.3.1-gcccoremkl-10.3.0-2021.2.0-Python-3.8.5/lib/python3.8/site-packages/xarray/backends/file_manager.py", line 205, in _acquire_with_cache_info
    file = self._opener(*self._args, **kwargs)
  File "netCDF4/_netCDF4.pyx", line 2358, in netCDF4._netCDF4.Dataset.__init__
  File "netCDF4/_netCDF4.pyx", line 1926, in netCDF4._netCDF4._ensure_nc_success
PermissionError: [Errno 13] Permission denied: b'/p/project/deepacf/intelliaq/leufen1/DATA/MLAIR/daily/AT0ZOE2_o3.nc'

First guess on error origin

Maybe the file wasn't properly closed? Try to close file. Otherwise try to remove the file if it is already existing.

would look as follows:

class DataHandlerSingleStation(AbstractDataHandler):
    ...
    def download_data(...):
        ...
        # convert df_all to xarray
        xarr = {k: xr.DataArray(v, dims=[time_dim, target_dim]) for k, v in df_all.items()}
        xarr = xr.Dataset(xarr).to_array(dim=iter_dim)
        if store_data_locally is True:
            # save locally as nc/csv file
+           if os.path.exists(file_name):
+               os.remove(file_name)
            xarr.to_netcdf(path=file_name)
            meta.to_csv(meta_file)
        return xarr, meta

Error origin

Solution

Edited by Ghost User