From 2835b1d2e27ad2d93ff98ce7d9622415f59f0b1c Mon Sep 17 00:00:00 2001 From: Ian Carroll Date: Tue, 25 Jun 2024 21:38:39 -0400 Subject: [PATCH 1/2] do not subclass --- earthaccess/store.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/earthaccess/store.py b/earthaccess/store.py index 147e6ce8..41a50d3e 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -25,8 +25,10 @@ logger = logging.getLogger(__name__) -class EarthAccessFile(fsspec.spec.AbstractBufferedFile): - def __init__(self, f: fsspec.AbstractFileSystem, granule: DataGranule) -> None: +class EarthAccessFile: + def __init__( + self, f: fsspec.spec.AbstractBufferedFile, granule: DataGranule + ) -> None: self.f = f self.granule = granule @@ -42,7 +44,7 @@ def __reduce__(self) -> Any: ) def __repr__(self) -> str: - return str(self.f) + return repr(self.f) def _open_files( From 9b3281d2b3bb9cf19e8836e98f8ec44a799d8e65 Mon Sep 17 00:00:00 2001 From: Ian Carroll Date: Thu, 27 Jun 2024 00:20:22 -0400 Subject: [PATCH 2/2] adds test, documentation --- CHANGELOG.md | 2 ++ docs/user-reference/store/earthaccessfile.md | 7 ++++ earthaccess/api.py | 8 ++--- earthaccess/store.py | 37 ++++++++++---------- mkdocs.yml | 1 + tests/unit/test_store.py | 10 ++++++ 6 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 docs/user-reference/store/earthaccessfile.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c5ee95ac..8eb2ac68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ ([#508](https://github.com/nsidc/earthaccess/issues/508)) * Create destination directory prior to direct S3 downloads, if it doesn't already exist ([#562](https://github.com/nsidc/earthaccess/issues/562)) +* Remove the base class on `EarthAccessFile` to fix method resolution + ([#610](https://github.com/nsidc/earthaccess/issues/610)) ## [v0.9.0] 2024-02-28 diff --git a/docs/user-reference/store/earthaccessfile.md b/docs/user-reference/store/earthaccessfile.md new file mode 100644 index 00000000..31b19c93 --- /dev/null +++ b/docs/user-reference/store/earthaccessfile.md @@ -0,0 +1,7 @@ +# Documentation for `EarthAccessFile` + +::: earthaccess.store.EarthAccessFile + options: + inherited_members: true + show_root_heading: true + show_source: false diff --git a/earthaccess/api.py b/earthaccess/api.py index 8522dbbb..aadd0579 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -10,7 +10,7 @@ from .auth import Auth from .results import DataCollection, DataGranule from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery -from .store import Store +from .store import EarthAccessFile, Store from .system import PROD, System from .utils import _validation as validate @@ -211,8 +211,8 @@ def download( def open( granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, -) -> List[AbstractFileSystem]: - """Returns a list of fsspec file-like objects that can be used to access files +) -> List[EarthAccessFile]: + """Returns a list of file-like objects that can be used to access files hosted on S3 or HTTPS by third party libraries like xarray. Parameters: @@ -221,7 +221,7 @@ def open( provider: e.g. POCLOUD, NSIDC_CPRD, etc. Returns: - a list of s3fs "file pointers" to s3 files. + A list of "file pointers" to remote (i.e. s3 or https) files. """ provider = _normalize_location(provider) results = earthaccess.__store__.open(granules=granules, provider=provider) diff --git a/earthaccess/store.py b/earthaccess/store.py index 41a50d3e..b1b4c175 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -26,9 +26,21 @@ class EarthAccessFile: + """Handle for a file-like object pointing to an on-prem or Earthdata Cloud granule.""" + def __init__( self, f: fsspec.spec.AbstractBufferedFile, granule: DataGranule ) -> None: + """EarthAccessFile connects an Earthdata search result with an open file-like object. + + No methods exist on the class, which passes all attribute and method calls + directly to the file-like object given during initialization. An instance of + this class can be treated like that file-like object itself. + + Parameters: + f: a file-like object + granule: a granule search result + """ self.f = f self.granule = granule @@ -51,7 +63,7 @@ def _open_files( url_mapping: Mapping[str, Union[DataGranule, None]], fs: fsspec.AbstractFileSystem, threads: Optional[int] = 8, -) -> List[fsspec.AbstractFileSystem]: +) -> List[EarthAccessFile]: def multi_thread_open(data: tuple) -> EarthAccessFile: urls, granule = data return EarthAccessFile(fs.open(urls), granule) @@ -303,17 +315,17 @@ def open( self, granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, - ) -> List[Any]: - """Returns a list of fsspec file-like objects that can be used to access files + ) -> List[EarthAccessFile]: + """Returns a list of file-like objects that can be used to access files hosted on S3 or HTTPS by third party libraries like xarray. Parameters: - granules: a list of granules(DataGranule) instances or list of URLs, - e.g. s3://some-granule - provider: an option + granules: a list of granule instances **or** list of URLs, e.g. `s3://some-granule`. + If a list of URLs is passed, we need to specify the data provider. + provider: e.g. POCLOUD, NSIDC_CPRD, etc. Returns: - A list of s3fs "file pointers" to s3 files. + A list of "file pointers" to remote (i.e. s3 or https) files. """ if len(granules): return self._open(granules, provider) @@ -325,17 +337,6 @@ def _open( granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, ) -> List[Any]: - """Returns a list of fsspec file-like objects that can be used to access files - hosted on S3 or HTTPS by third party libraries like xarray. - - Parameters: - granules: a list of granules(DataGranule) instances or list of URLs, - e.g. s3://some-granule - provider: an option - - Returns: - A list of s3fs "file pointers" to s3 files. - """ raise NotImplementedError("granules should be a list of DataGranule or URLs") @_open.register diff --git a/mkdocs.yml b/mkdocs.yml index ab48481f..f962c82f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -97,6 +97,7 @@ nav: - "Granule Queries": "user-reference/granules/granules-query.md" - "Granule Results": "user-reference/granules/granules.md" - Store: + - "EarthAccessFile": "user-reference/store/earthaccessfile.md" - "Store": "user-reference/store/store.md" - Auth: - "Auth": "user-reference/auth/auth.md" diff --git a/tests/unit/test_store.py b/tests/unit/test_store.py index 4da4e0e3..1ddb0664 100644 --- a/tests/unit/test_store.py +++ b/tests/unit/test_store.py @@ -7,6 +7,7 @@ import responses import s3fs from earthaccess import Auth, Store +from earthaccess.store import EarthAccessFile class TestStoreSessions(unittest.TestCase): @@ -129,3 +130,12 @@ def test_store_can_create_s3_fsspec_session(self): store.get_s3fs_session() return None + + +def test_earthaccess_file_getattr(): + fs = fsspec.filesystem("memory") + with fs.open("/foo", "wb") as f: + earthaccess_file = EarthAccessFile(f, granule="foo") + assert f.tell() == earthaccess_file.tell() + # cleanup + fs.store.clear()