from misc import Timer,Status,setup_logger from pySmartDL import SmartDL as DM from pySmartDL import utils from io import StringIO import requests import threading import time class PayLoad: # Recheck period in seconds RECHECK_PERIOD = 120 ESA_CREDENTIALS = () __thread_rlock = threading.RLock() def __init__(self, key, download_link, output_dir, filename, md5_checksum,payload_size,status=Status.Offline,ext=".zip"): self.__key = key self.__filename = filename self.__download_link = download_link self.__output_dir = output_dir self.__credentials = PayLoad.ESA_CREDENTIALS self.__status = status self.__logger_stream = StringIO() self.__logger = setup_logger(key,self.__logger_stream) self.__downloader = None self.__downloader_ready = False self.__queued_for_download = True self.__is_hash_correct = [] self.__md5_checksum = md5_checksum.upper() self.__payload_size = payload_size self.__ext = ext self.__hash_check_thread = None self.__request_thread = None self.__recheck_thread = None self.__fail_retries=3 # seconds to wait before re-downloading self.__fail_retry_in = 60 self.__fail_retry_at = None self.__retry_in_multiplier = 2 @property def key(self): return self.__key @property def filename(self): return self.__filename @property def product_short_name(self): prodName = self.__filename return prodName[0:4]+prodName[17:32] @property def download_link(self): return self.__download_link @property def output_dir(self): return self.__output_dir @property def full_path(self): return self.__output_dir+"/"+self.__filename+self.__ext @property def size(self): return self.__payload_size @property def esa_credentials(self): return self.__credentials @esa_credentials.setter def esa_credentials(self,value): if type(value) is not tuple and len(value) != 2: raise ValueError('Credentials should be a tuple of length 2') else: self.__credentials = value @property def downloader(self): return self.__downloader @property def status(self): return self.__status @status.setter def status(self,value): self.__status = value @property def log(self): return self.__logger @property def log_text(self): return self.__logger_stream @property def queued_for_download(self): return self.__queued_for_download @queued_for_download.setter def queued_for_fownload(self,value): self.__queued_for_download = value def toggle_download(self): self.__queued_for_download = not self.__queued_for_download def configure_downloader(self): if type(self.__credentials) is not tuple and len(self.__credentials) != 2: raise ValueError('Credentials Not setup correctly') self.__downloader = DM(self.__download_link,dest=self.__output_dir+"/"+self.__filename+self.__ext, progress_bar=False,fix_urls=False,logger=self.__logger,hash_checksum=self.__md5_checksum,hash_algo='md5',username=self.__credentials[0],password=self.__credentials[1],threads=1) self.__downloader.add_basic_authentication(self.__credentials[0],self.__credentials[1]) self.__downloader_ready = True def download(self): if self.__downloader_ready and self.__status is Status.Online: try: self.__downloader.start(blocking=False) self.__status = Status.Downloading self.__is_hash_correct.append(-2) except: self.__status = Status.Failed if self.__fail_retries > 0: self.__fail_retries = self.__fail_retries - 1 self.__fail_retry_at = time.time() + self.__fail_retry_in self.__logger.warning("Download failed - Retries left [{}] - Next retry in: {}".format(self.__fail_retries,utils.time_human(self.__fail_retry_in,fmt_short=True))) self.__fail_retry_in *= self.__retry_in_multiplier else: self.__logger.error("Download failed - No retries left\nEnd Log\n{}".format("#"*20)) else: if not self.__downloader_ready: self.__logger.warning("Could not start downlad because the download manager is not Ready") elif self.__status is Status.Downloading: self.__logger.warning("could not start download because download is already in progress") def attempt_request(self,main_timer:Timer): self.__thread_rlock.acquire() if main_timer.has_elapsed(PayLoad.RECHECK_PERIOD): try: self.__logger.info("Requesting") r = requests.get(self.__download_link,auth=self.__credentials, verify=True,stream=True, timeout=10) except Exception as e: self.__logger.error("Could not attempt request:\n{}".format(str(e))) self.__thread_rlock.release() return if r.status_code == 202: self.__status = Status.Requested self.__logger.info("Image has been requested") elif r.status_code == 200: self.__status = Status.Online self.__logger.info("Image is Online - Queued for download") elif r.status_code == 403: self.__status = Status.Pending self.__logger.info("Requests exceed user quota") main_timer.tick() self.__thread_rlock.release() else: self.__thread_rlock.release() def check_online(self,main_timer): self.__thread_rlock.acquire() if main_timer.has_elapsed(PayLoad.RECHECK_PERIOD/3): try: r = requests.get(self.__download_link.replace('$value','?$format=json'),auth=self.__credentials, verify=True,stream=True, timeout=10) if r.status_code == 200 : self.update_checksum_from_meta(r.json()['d']) self.update_status_from_meta(r.json()['d']) self.__logger.info("Checking Online availability") main_timer.tick() self.__thread_rlock.release() except Exception as e: self.__logger.info("Could not check Online availability:\n{}".format(str(e))) self.__thread_rlock.release() return else: self.__thread_rlock.release() def verify_hash_threaded(self): self.__logger.info("Calculating Hash...") if self.__is_hash_correct[-1] == -2: self.__hash_check_thread = threading.Thread(target=self.__verify_hash_thread_target,args=()) self.__hash_check_thread.daemon = True self.__hash_check_thread.start() self.__is_hash_correct[-1] = -1 def verify_hash(self,md5_checksum): return 1 hashValue = utils.get_file_hash('md5',self.__output_dir+"/"+self.__filename+self.__ext) hashValue = hashValue.upper() if hashValue == self.__md5_checksum.upper() : return 1 else: return 0 def __verify_hash_thread_target(self): self.__is_hash_correct[-1] = self.verify_hash(self.__md5_checksum) def auto_update_status(self,main_timer:Timer): if (self.__status < Status.Requested and ((self.__request_thread and self.__request_thread.is_alive() is not True) or self.__request_thread is None)): self.__request_thread = threading.Thread(target=self.attempt_request,args=(main_timer,)) self.__request_thread.daemon = True self.__request_thread.start() elif (self.__status == Status.Requested and ((self.__recheck_thread and self.__recheck_thread.is_alive() is not True) or self.__recheck_thread is None)): self.__logger.info("we will see if online") self.__recheck_thread = threading.Thread(target=self.check_online,args=(main_timer,)) self.__recheck_thread.daemon = True self.__recheck_thread.start() # elif self.__status == Status.Online: # self.download() # self.__status = Status.Downloading elif self.__status == Status.Downloading: if self.__downloader.isFinished(): if self.__downloader.isSuccessful(): self.__status = Status.Hash self.verify_hash_threaded() else: self.__status = Status.Failed if self.__fail_retries > 0: self.__fail_retries = self.__fail_retries - 1 self.__fail_retry_at = time.time() + self.__fail_retry_in self.__logger.warning("Download failed - Retries left [{}] - Next retry in: {}".format(self.__fail_retries,utils.time_human(self.__fail_retry_in,fmt_short=True))) self.__fail_retry_in *= self.__retry_in_multiplier else: self.__logger.error("Download failed - No retries left\nEnd Log\n{}".format("#"*20)) elif self.__status == Status.Downloaded: self.__status = Status.Hash self.__is_hash_correct.append(-2) self.verify_hash_threaded() elif self.__status == Status.Failed: if time.time() >= self.__fail_retry_at: self.__logger.info("Resetting Downloader and requeuing for download") self.configure_downloader() self.__status = Status.Offline elif self.__status == Status.Hash: ch = self.__is_hash_correct[-1] if ch == 1: self.__status = Status.Succeeded self.__logger.info("Hash matches\nEnd Log\n{}".format("#"*20)) elif ch == 0: self.__status = Status.Corrupted self.__logger.info("Hash doesn't match.\nDownloaded file is corrupted") def update_checksum_from_meta(self,responseJson): self.__md5_checksum = responseJson['Checksum']['Value'] def update_status_from_meta(self,responseJson): if responseJson['Online']: self.__status = Status.Online self.__logger.info("Image is Online") else: self.__logger.info("Image is Online?: %s"%responseJson['Online'])