Skip to content

Commit

Permalink
Refactor zero sized file downloading (modelscope#991)
Browse files Browse the repository at this point in the history
  • Loading branch information
tastelikefeet committed Sep 14, 2024
1 parent 4c518db commit 74d97ea
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 2 deletions.
10 changes: 8 additions & 2 deletions modelscope/hub/file_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -461,17 +461,23 @@ def http_get_model_file(
unit='B',
unit_scale=True,
unit_divisor=1024,
total=file_size,
total=file_size if file_size > 0 else 1,
initial=0,
desc='Downloading [' + file_name + ']',
)
if file_size == 0:
# Avoid empty file server request
with open(temp_file_path, 'w+'):
progress.update(1)
progress.close()
break
partial_length = 0
if os.path.exists(
temp_file_path): # download partial, continue download
with open(temp_file_path, 'rb') as f:
partial_length = f.seek(0, io.SEEK_END)
progress.update(partial_length)
if partial_length >= file_size > 0:
if partial_length >= file_size:
break
# closed range[], from 0.
get_headers['Range'] = 'bytes=%s-%s' % (partial_length,
Expand Down
31 changes: 31 additions & 0 deletions tests/hub/test_hub_empty_file.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import os.path
import shutil
import tempfile
import unittest

from modelscope import snapshot_download


class HubEmptyFile(unittest.TestCase):

def setUp(self):
temporary_dir = tempfile.mkdtemp()
self.work_dir = temporary_dir

def tearDown(self):
shutil.rmtree(self.work_dir, ignore_errors=True)

def test_download_empty_file(self):
model_dir = snapshot_download(
'tastelikefeet/test_empty_download', cache_dir=self.work_dir)
self.assertTrue(model_dir is not None)
self.assertTrue(os.path.exists(os.path.join(model_dir, '1.txt')))
self.assertTrue(
os.path.exists(os.path.join(model_dir, 'configuration.json')))
self.assertTrue(os.path.exists(os.path.join(model_dir, 'init.py')))
self.assertTrue(os.path.exists(os.path.join(model_dir, 'README.md')))


if __name__ == '__main__':
unittest.main()

0 comments on commit 74d97ea

Please sign in to comment.