You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

59 lines
1.8 KiB

from _io import BufferedWriter
from argparse import ArgumentParser
from datetime import datetime, timedelta
from pathlib import Path
from shutil import copyfileobj
from time import sleep
from zipfile import ZipFile
import logging
import requests
def file_download(*, url: str, file_open: BufferedWriter) -> None:
response = requests.head(url)
content_length = response.headers["content-length"]
file_end = file_open.tell()
headers = {"range": f"bytes={file_end}-{content_length}"}
logging.info(headers)
try:
with requests.get(url, headers=headers, stream=True) as response:
copyfileobj(response.raw, file_open)
except Exception as error:
logging.error(error)
sleep(1)
file_download(url=url, file_open=file_open)
def zipfile_download():
description = """
Download large zipfile from URL, retry at offset if interrupted, check zip.
"""
parser = ArgumentParser(description=description)
parser.add_argument("-v", "--verbose", action="count")
parser.add_argument("-url", "--url", required=True, help="URL to zipfile")
parser.add_argument("-o", "--path", required=True, help="path to write zipfile")
args = parser.parse_args()
level = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(level=level)
logging.info(args)
today = datetime.utcnow().strftime("%Y%m%d")
logging.info(today)
path = Path(args.path)
if path.exists():
logging.info(f'output path exists, resuming: "{path}"')
with path.open("wb") as file_open:
file_download(url=args.url, file_open=file_open)
with ZipFile(path) as zip_file:
file_list = zip_file.namelist()
if not file_list:
raise Exception(f'zipfile_empty "{args.url}"')
if __name__ == "__main__":
zipfile_download()