implement chunking for _get_file()

ipfsspec/async_ipfs.py

@@ -4,6 +4,7 @@
 import weakref
 from functools import lru_cache
 from pathlib import Path
+from contextlib import asynccontextmanager
 import warnings
 
 import asyncio
@@ -120,6 +121,17 @@ async def cat(self, path, session):
             self._raise_not_found_for_status(res, path)
             return await res.read()
 
+    @asynccontextmanager
+    async def iter_chunked(self, path, session, chunk_size):
+        res = await self.get(path, session)
+        async with res:
+            self._raise_not_found_for_status(res, path)
+            try:
+                size = int(res.headers["content-length"])
+            except (ValueError, KeyError):
+                size = None
+            yield size, res.content.iter_chunked(chunk_size)
+
     async def ls(self, path, session, detail=False):
         res = await self.get(path, session, headers={"Accept": "application/vnd.ipld.raw"}, params={"format": "raw"})
         self._raise_not_found_for_status(res, path)
@@ -298,19 +310,21 @@ async def _cat_file(self, path, start=None, end=None, **kwargs):
     async def _get_file(
         self, rpath, lpath, chunk_size=5 * 2**20, callback=DEFAULT_CALLBACK, **kwargs
     ):
-        # TODO: implement chunked retrieval
         logger.debug(rpath)
+        rpath = self._strip_protocol(rpath)
+        session = await self.set_session()
 
         if isfilelike(lpath):
             outfile = lpath
         else:
             outfile = open(lpath, "wb")  # noqa: ASYNC101, ASYNC230
 
         try:
-            content = await self._cat_file(rpath)
-            outfile.write(content)
-            callback.set_size(len(content))
-            callback.relative_update(len(content))
+            async with self.gateway.iter_chunked(rpath, session, chunk_size) as (size, chunks):
+                callback.set_size(size)
+                async for chunk in chunks:
+                    outfile.write(chunk)
+                    callback.relative_update(len(chunk))
         finally:
             if not isfilelike(lpath):
                 outfile.close()