You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
59 lines
1.8 KiB
59 lines
1.8 KiB
from _io import BufferedWriter
|
|
from argparse import ArgumentParser
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from shutil import copyfileobj
|
|
from time import sleep
|
|
from zipfile import ZipFile
|
|
import logging
|
|
import requests
|
|
|
|
|
|
def file_download(*, url: str, file_open: BufferedWriter) -> None:
|
|
response = requests.head(url)
|
|
content_length = response.headers["content-length"]
|
|
file_end = file_open.tell()
|
|
headers = {"range": f"bytes={file_end}-{content_length}"}
|
|
logging.info(headers)
|
|
|
|
try:
|
|
with requests.get(url, headers=headers, stream=True) as response:
|
|
copyfileobj(response.raw, file_open)
|
|
except Exception as error:
|
|
logging.error(error)
|
|
sleep(1)
|
|
file_download(url=url, file_open=file_open)
|
|
|
|
|
|
def zipfile_download():
|
|
description = """
|
|
Download large zipfile from URL, retry at offset if interrupted, check zip.
|
|
"""
|
|
parser = ArgumentParser(description=description)
|
|
parser.add_argument("-v", "--verbose", action="count")
|
|
parser.add_argument("-url", "--url", required=True, help="URL to zipfile")
|
|
parser.add_argument("-o", "--path", required=True, help="path to write zipfile")
|
|
args = parser.parse_args()
|
|
level = logging.DEBUG if args.verbose else logging.INFO
|
|
logging.basicConfig(level=level)
|
|
logging.info(args)
|
|
today = datetime.utcnow().strftime("%Y%m%d")
|
|
logging.info(today)
|
|
|
|
path = Path(args.path)
|
|
|
|
if path.exists():
|
|
logging.info(f'output path exists, resuming: "{path}"')
|
|
|
|
with path.open("wb") as file_open:
|
|
file_download(url=args.url, file_open=file_open)
|
|
|
|
with ZipFile(path) as zip_file:
|
|
file_list = zip_file.namelist()
|
|
|
|
if not file_list:
|
|
raise Exception(f'zipfile_empty "{args.url}"')
|
|
|
|
|
|
if __name__ == "__main__":
|
|
zipfile_download()
|