38 lines
1.4 KiB
Python
38 lines
1.4 KiB
Python
"""Исходящий pull файла из ВМ-буфера с ретраями и лимитом размера."""
|
|
|
|
import logging
|
|
import time
|
|
|
|
from .config import PULL_RETRIES, PULL_RETRY_DELAY
|
|
|
|
|
|
log = logging.getLogger("upload.upload_refs.pull")
|
|
|
|
|
|
def pull_file(client, url: str, max_bytes: int,
|
|
retries: int = PULL_RETRIES, delay: float = PULL_RETRY_DELAY,
|
|
timeout: int = 120, sid: str = None, name: str = None) -> bytes:
|
|
"""Забрать файл потоково, не принимая тело больше ``max_bytes``."""
|
|
|
|
last_error = None
|
|
for attempt in range(retries):
|
|
try:
|
|
response = client.get(url, stream=True, timeout=timeout)
|
|
response.raise_for_status()
|
|
chunks = []
|
|
total = 0
|
|
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
|
if not chunk:
|
|
continue
|
|
total += len(chunk)
|
|
if total > max_bytes:
|
|
raise ValueError("file exceeds maxFileBytes")
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
except Exception as error:
|
|
last_error = error
|
|
log.warning("pull: attempt %d/%d failed sid=%s file=%r: %r",
|
|
attempt + 1, retries, sid, name, error)
|
|
if attempt + 1 < retries:
|
|
time.sleep(delay)
|
|
raise last_error if last_error else RuntimeError("pull failed") |