import npyscreen import sys from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt,SentinelAPIError as sErr from configparser import ConfigParser import os from pySmartDL import utils import threading import requests from dateutil import parser as dp from misc import Timer,Status from payloadmanager import PayLoad from npyscreenoverrides import CGrid from collections import OrderedDict import time # Main TUI class MainTUI(npyscreen.NPSAppManaged): def onStart(self): # npyscreen.setTheme(npyscreen.Themes.BlackOnWhiteTheme) self.payloads = OrderedDict() self.addForm("MAIN", ConfigTUI, name="Configuration") self.addForm("DM", DMTUI,name="Download Manager") # General Config TUI class ConfigTUI(npyscreen.ActionFormWithMenus): esauser = "" esapass = "" geopath = "" outpath = "" dfrom = None dto = None row_to_key = dict() inline_readme = "" on_toggle_select_all = True concurrent_downloads = 1 count_selected = 0 def activate(self): self.edit() # self.parentApp.setNextForm("DM") def create(self): # Get the space used by the form y, x = self.useable_space() self.keypress_timeout = 1 self.OK_BUTTON_TEXT = "Exit" self.CANCEL_BUTTON_TEXT = "Download" self.h_display_help ## Try load config file if os.path.exists("config.ini"): parser = ConfigParser() parser.read('config.ini') if parser.has_option('ESA','username'): self.esauser = parser.get('ESA','username') if parser.has_option('ESA','password'): self.esapass = parser.get('ESA','password') if parser.has_option("GEOJSON","path"): self.geopath = parser.get("GEOJSON","path") if parser.has_option("Dates","from"): self.dfrom = dp.parse(parser.get("Dates","from")) if parser.has_option("Dates","to"): self.dto = dp.parse(parser.get("Dates","to")) if parser.has_option("OUTPUT","path"): self.outpath = parser.get("OUTPUT","path") if parser.has_option("OPTIONS","concurrent-downloads"): self.concurrent_downloads = parser.getint("OPTIONS","concurrent-downloads") if os.path.exists("inline_readme.txt"): with open("inline_readme.txt", 'r',encoding="utf-8") as readme: self.inline_readme = readme.read() ########## # MENUS # ########## self.mainMenu = self.add_menu(name="Shortcuts", shortcut="^S") self.mainMenu.addItemsFromList([ ("Save Configuration", self.h_saveConfiguration,"^C"), ("Search For Images", self.searchButtonPress,"^S" ), ("Start Download", self.on_cancel, "^D"), ("Exit Application", self.on_ok, "^E"), ]) ########## # BOXES # ########## self.credentialsBox = self.add(npyscreen.BoxTitle, _contained_widget = npyscreen.TitleFixedText, relx = 1, rely = 1, editable = False, height = 5, width = 75, name = 'Esa Credentials') self.SelectionBox = self.add(npyscreen.BoxTitle, _contained_widget = npyscreen.TitleFixedText, relx = 1, rely = 6, editable = False, height = 6, width = 75, name = 'Images Date & Location') self.ReadmeBox = self.add(npyscreen.BoxTitle, _contained_widget = npyscreen.TitleFixedText, relx = 1, rely = 24, editable = False, height = 11, width = 75, name = '// Readme') self.GridBox = self.add(npyscreen.BoxTitle, _contained_widget = npyscreen.TitleFixedText, relx = 81, rely = 1, editable = False, width = 40, height = 34, name = 'Available Images', footer = "press 's'/'d' to select/deselect all") ########## #WIDGETS # ########## self.userName = self.add(npyscreen.TitleText, value = self.esauser, name = 'UserName: ', use_two_lines = False, rely = self.credentialsBox.rely + 2, relx = self.credentialsBox.relx + 1, width = 30, height = 1, begin_entry_at = 11, editable = True ) self.password = self.add(npyscreen.TitlePassword, value = self.esapass, name = 'Password: ', use_two_lines = False, rely = self.credentialsBox.rely + 3, relx = self.credentialsBox.relx + 1, width = 30, height = 1, begin_entry_at = 11, editable = True ) self.geoJSonPath = self.add(npyscreen.TitleFilenameCombo, value = self.geopath, name = 'GeoJson Path: ', use_two_lines = False, rely = self.SelectionBox.rely + 2, relx = self.SelectionBox.relx + 1, width = 74, height=1, begin_entry_at = 14, editable = True ) self.geoPath = self.add(npyscreen.TitleFilenameCombo, value = self.outpath, name = 'Save Path: ', use_two_lines = False, rely = self.SelectionBox.rely + 3, relx = self.SelectionBox.relx + 1, width = 74, height=1, begin_entry_at = 14, editable = True, select_dir=True ) self.dateFrom = self.add(npyscreen.TitleDateCombo, value = self.dfrom, name = 'Date From: ', use_two_lines = False, rely = self.SelectionBox.rely + 4, relx = self.SelectionBox.relx + 1, width = 30, height=1, begin_entry_at = 11, editable = True ) self.dateTo = self.add(npyscreen.TitleDateCombo, value = self.dto, name = 'Date To: ', use_two_lines = False, rely = self.SelectionBox.rely + 4, relx = self.SelectionBox.relx + 40, width = 30, height=1, begin_entry_at = 11, editable = True ) self.ImageSearchButton = self.add(npyscreen.ButtonPress, name = '', use_two_lines = False, rely = self.SelectionBox.rely + 6, relx = self.SelectionBox.relx + 1, when_pressed_function=self.searchButtonPress ) self.ConfigSaveButton = self.add(npyscreen.ButtonPress, name = '', use_two_lines = False, rely = self.SelectionBox.rely + 6, relx = self.SelectionBox.relx + 22, when_pressed_function=self.h_saveConfiguration ) self.InlineReadme = self.add(npyscreen.MultiLineEdit, name = '// Readme', value = self.inline_readme, relx = self.ReadmeBox.relx + 1, rely = self.ReadmeBox.rely + 1, editable=False, width = 70, ) self.ImagesGird = self.add(CGrid, columns = 3, col_titles = ['( )','Identifier','Status'], select_whole_line = True, rely = self.GridBox.rely + 1, relx = self.GridBox.relx + 1, editable = False, col_widths = [5,20,8], width = 33, height = 31, slow_scroll = True, name = "imgGrid" ) self.GridMoreLabel = self.add(npyscreen.FixedText, value = '- more -', relx = self.GridBox.relx+1, rely = self.GridBox.rely + self.GridBox.height - 2, color='CONTROL', editable = False, hidden = True) self.ImagesGird.add_handlers({"x": self.h_select, "s": self.h_toggle_all_select, "d": self.h_toggle_all_deselect }) def while_waiting(self): grid = self.ImagesGird if(grid.values and ((grid.height + grid.begin_row_display_at - 2) < (len(grid.values)))): self.GridMoreLabel.hidden = False else: self.GridMoreLabel.hidden = True self.display() def on_ok(self): notify_result = npyscreen.notify_ok_cancel("Are you sure you want to exit ?", title='Exit ?',form_color='CONTROL',wrap=True,editw = 1) if(notify_result): sys.exit() def on_cancel(self): if not os.path.isdir(self.geoPath.value): warning_message = "Path: {}\nIs not a directory. Please choose a valid directory path".format(self.geoPath.value) npyscreen.notify_confirm(warning_message, title='Invalid Path',form_color='DANGER',wrap=True,editw = 1) elif not os.path.exists(self.geoPath.value): warning_message = "Path: {}\nDoes not exist. Create Directory ?".format(self.geoPath.value) notify_result = npyscreen.notify_yes_no(warning_message, title='Directory not Found',form_color='CAUTION',wrap=True,editw = 1) if(notify_result): try: os.mkdir(self.geoPath.value) npyscreen.notify_confirm("Directory Created", title='Directory Created',form_color='CAUTION',wrap=True,editw = 1) except Exception: npyscreen.notify_confirm("Could not Create Directory", title='Directory Creatation Failed',form_color='Critical',wrap=True,editw = 1) elif len(self.parentApp.payloads.keys()) > 0: DMForm = self.parentApp.getForm("DM") DMForm.concurrent_downloads = self.concurrent_downloads DMForm.count_selected = self.count_selected DMForm.PopulateGrid() self.parentApp.switchForm("DM") elif len(self.parentApp.payloads.keys()) == 0: npyscreen.notify_confirm("Nothing to Download", title='Empty List',form_color='CAUTION',wrap=True,editw = 1) def searchButtonPress(self): npyscreen.notify("Searching for Images between\n {} and {}".format(self.dateFrom.value,self.dateTo.value), title='Searching...') try: api = SentinelAPI(self.userName.value, self.password.value, 'https://scihub.copernicus.eu/dhus') footprint = geojson_to_wkt(read_geojson(self.geoJSonPath.value)) products = api.query(footprint,date=(self.dateFrom.value,self.dateTo.value),platformname='Sentinel-1',producttype='GRD') self.ImagesGird.values = [] PayLoad.ESA_CREDENTIALS = self.userName.value,self.password.value payloads = self.parentApp.payloads rowi = 0 for key in sorted(products, key=lambda k: products[k]['ingestiondate']): prodName = products[key]['identifier'] prodName = prodName[0:4]+prodName[17:32] status = Status.Offline row = [] product_info = api.get_product_odata(key) row.append('(X)') self.count_selected +=1 row.append(prodName) if(product_info['Online']): row.append('Online') status = Status.Online else: row.append('Offline') self.ImagesGird.values.append(row) payloads[key] = [PayLoad(key,products[key]['link'],self.geoPath.value,products[key]['identifier'],product_info['md5'],product_info['size'],status=status),-1] self.row_to_key[rowi] = key rowi +=1 if rowi > 0: self.ImagesGird.editable = True self.ImagesGird.begin_row_display_at = 0 self.ImagesGird.begin_col_display_at = 0 self.ImagesGird.edit_cell = [0, 0] self.h_selectGridWidget() self.display() # Sentinel Sat Api errors except sErr as e: npyscreen.notify_confirm(str(e), title='Server Error...',form_color='DANGER',wrap=True,editw = 1) # requests raised errors except requests.exceptions.HTTPError as e: npyscreen.notify_confirm(str(e), title='HTTP Error',form_color='DANGER',wrap=True,editw = 1) except requests.exceptions.ConnectionError as e: npyscreen.notify_confirm(str(e), title='Connection Error',form_color='DANGER',wrap=True,editw = 1) except requests.exceptions.Timeout as e: npyscreen.notify_confirm(str(e.response), title='Timeout Error [{}]'.format(1),form_color='DANGER',wrap=True,editw = 1) # Other errors except Exception as e: npyscreen.notify_confirm(str(e), title='Error...',form_color='DANGER',wrap=True,editw = 1) def h_select(self, inpt): if(self.ImagesGird.edit_cell != -1): row = self.ImagesGird.edit_cell[0] col = self.ImagesGird.edit_cell[1] (pl,value) = self.parentApp.payloads[self.row_to_key[row]] pl.toggle_download() if(self.ImagesGird.values[row][0] == '(X)'): self.ImagesGird.values[row][0] = '( )' self.count_selected -=1 else: self.ImagesGird.values[row][0] = '(X)' self.count_selected +=1 def h_toggle_all_deselect(self,inpt): self.a_toggle_all_select(False) def h_toggle_all_select(self,inpt): self.a_toggle_all_select(True) def a_toggle_all_select(self,select=True): select_text = "(X)" select_img = True add_to_selected = 1 if not select: select_text = "( )" select_img = False self.on_toggle_select_all = True add_to_selected = -1 else: self.on_toggle_select_all = False add_to_selected = 1 for row in range(0,len(self.ImagesGird.values)): pl,value = self.parentApp.payloads[self.row_to_key[row]] pl.queued_for_fownload = select_img self.ImagesGird.values[row][0] = select_text self.count_selected += add_to_selected def h_saveConfiguration(self): config_file = [ "[OPTIONS]\n", "; How many images will the program download at the same time\n", "concurrent-download=2", ";The username and password of your ESA account (https://scihub.copernicus.eu/dhus/#/home)\n", "[ESA]\n", "username={}\n".format(self.userName.value), "password={}\n".format(self.password.value), "\n", ";The location of the GeoJSON file\n", ";Can be generated from http://geojson.io\n", "[GEOJSON]\n", "path={}\n".format(self.geoJSonPath.value), "\n", ";Images date range\n", "[Dates]\n", "from={}\n".format(self.dateFrom.value.strftime('%Y-%m-%d')), "to={}\n".format(self.dateTo.value.strftime('%Y-%m-%d')), "\n", ";Output folder of the downloaded images\n", "[OUTPUT]\n", "path={}\n".format(self.geoPath.value), ] with open('config.ini', 'w') as fp: fp.writelines(config_file) def h_selectGridWidget(self): r = list(range(0, len(self._widgets__))) for n in r: if self._widgets__[n].name == "imgGrid": if self.editw > n : self._widgets__[self.editw].h_exit_up(True) else: self._widgets__[self.editw].h_exit_up(True) self._widgets__[self.editw].editing = False try: self._widgets__[self.editw].entry_widget.editing = False except: pass self.editw = n self._widgets__[self.editw].edit() self.display() class DMTUI(npyscreen.ActionFormMinimal): rowToKey = dict() checker_requester_thread = None thread_stop = False currentKey = None main_timer = Timer() current_concurrent_downloads = 0 concurrent_downloads = 1 concurrent_list = list() count_selected = 0 max_hash = 3 current_hash = 0 hash_keys = dict() progress_look_feel = ('-','#') download_cooldown = 20 download_timer = Timer() def activate(self): self.edit() self.parentApp.setNextForm(None) self.keypress_timeout = 1 def create(self): # Get the space used by the form y, x = self.useable_space() self.keypress_timeout = 1 self.OK_BUTTON_TEXT = "Exit" self.RecheckBox = self.add(npyscreen.BoxTitle, _contained_widget = npyscreen.TitleFixedText, relx = 1, rely = 1, editable = False, width = 104, height = 4, name = 'Recheck Counters') self.RecheckCounter = self.add(npyscreen.TitleText, value = '-', name = 'Recheck in: ', use_two_lines = False, rely = self.RecheckBox.rely + 2, relx = self.RecheckBox.relx + 2, height = 1, begin_entry_at = 13, editable = False, width = 50 ) self.GridBox = self.add(npyscreen.BoxTitle, _contained_widget = npyscreen.TitleFixedText, relx = 1, rely = 5, editable = False, width = 104, height = 18, name = 'Download Status') self.ConsoleBox = self.add(npyscreen.BoxTitle, _contained_widget = npyscreen.TitleFixedText, relx = 1, rely = 24, editable = False, width = 104, height = 15, name = 'Console') self.Console = self.add(npyscreen.Pager, relx = self.ConsoleBox.relx+1, rely = self.ConsoleBox.rely+1, width = 102, height = 14, max_height=13, slow_scroll=True, autowrap=True) self.Console.add_handlers({"^P": self.h_console_previous, "^N": self.h_console_next }) self.ImagesGird = self.add(CGrid, columns = 5, col_titles = ['Identifier','Status','Progress','Rate','ETA'], select_whole_line = True, rely = self.GridBox.rely + 1, relx = self.GridBox.relx + 1, editable = False, col_widths = [20,13,42,10,15,], width = 100, height = 15, slow_scroll = True, ) self.GridMoreLabel = self.add(npyscreen.FixedText, value = '- more -', relx = self.GridBox.relx+1, rely = self.GridBox.rely + self.GridBox.height - 2, color='CONTROL', editable = False, hidden = True) self.add_handlers({"^S":self.h_check_checker}) def h_check_checker(self,inpt): for t in threading.enumerate(): if t.name == "Checker Thread": if t.is_alive(): npyscreen.notify_confirm("Checker Thread is still running") return else: npyscreen.notify_confirm("Checker Thread is dead") return npyscreen.notify_confirm("Couldn't find thread") def while_waiting(self): if self.ImagesGird.edit_cell != -1 : row = self.ImagesGird.edit_cell[0] log = self.parentApp.payloads[self.rowToKey[row]][0].log_text self.Console.clear() self.Console.values = log.getvalue().splitlines(True) self.Console.display() self.RecheckCounter.value = utils.time_human(self.main_timer.tock(),fmt_short=True) # show or hide the - more - button in the grid grid = self.ImagesGird if(grid.values and ((grid.height + grid.begin_row_display_at - 2) < (len(grid.values)))): self.GridMoreLabel.hidden = False # Downloads Manager # if we are not downloading, find first online file and download it if not self.currentKey and (not self.download_timer.running() or self.download_timer.tock() > self.download_cooldown): for key,(payload,iRow) in self.parentApp.payloads.items(): if not payload.queued_for_download: continue if payload.status == Status.Online: if (self.current_hash < self.max_hash and os.path.exists(payload.full_path) and (os.path.getsize(payload.full_path)==payload.size)): payload.status = Status.Downloaded if not key in self.hash_keys: self.current_hash += 1 self.hash_keys[key] = "" else: self.currentKey = key self.current_concurrent_downloads = self.current_concurrent_downloads + 1 payload.log.info("Attempting Download") payload.download() self.ImagesGird.values[iRow][1] = str(payload.status) break elif self.currentKey: (payload,iRow) = self.parentApp.payloads[self.currentKey] if payload.status == Status.Downloading: self.ImagesGird.values[iRow][2] = payload.downloader.get_progress_bar(length=15) + " ({} of {})".format(payload.downloader.get_dl_size(human=True),utils.sizeof_human(payload.size)) self.ImagesGird.values[iRow][3] = payload.downloader.get_speed(human=True) self.ImagesGird.values[iRow][4] = utils.time_human(payload.downloader.get_eta(),fmt_short=True) else: self.download_timer.tick() if payload.status == Status.Downloaded: self.ImagesGird.values[iRow][1] = str(payload.status) self.ImagesGird.values[iRow][2] = "Downloaded {} of {} in {}".format(utils.sizeof_human(payload.downloader.get_dl_size()),utils.sizeof_human(payload.downloader.filesize),utils.time_human(payload.downloader.get_dl_time(),fmt_short=True)) self.ImagesGird.values[iRow][3] = " " self.ImagesGird.values[iRow][4] = " " else: self.ImagesGird.values[iRow][1] = str(payload.status) self.ImagesGird.values[iRow][2] = "Downloaded {} of {} in {}".format(utils.sizeof_human(payload.downloader.get_dl_size()),utils.sizeof_human(payload.downloader.filesize),utils.time_human(payload.downloader.get_dl_time(),fmt_short=True)) self.ImagesGird.values[iRow][3] = " " self.ImagesGird.values[iRow][4] = " " self.currentKey = None for key,(payload,iRow) in self.parentApp.payloads.items(): if not payload.queued_for_download: continue if payload.status == Status.Downloading or payload.status == Status.Hash: payload.auto_update_status(self.main_timer) elif payload.status == Status.Downloaded and self.current_hash < self.max_hash: self.ImagesGird.values[iRow][1] = 'Downloaded' self.ImagesGird.values[iRow][2] = "" payload.auto_update_status(self.main_timer) elif payload.status == Status.Hash: self.ImagesGird.values[iRow][1] = str(payload.status) elif payload.status == Status.Succeeded or payload.status == Status.Corrupted: self.ImagesGird.values[iRow][1] = str(payload.status) if payload.status == Status.Corrupted: self.ImagesGird.values[iRow][2] = 'file downloaded but seems corrupted' else: self.ImagesGird.values[iRow][2] = 'file downloaded successfully' if key in self.hash_keys: self.current_hash -=1 del self.hash_keys[key] elif payload.status == Status.Pending: self.ImagesGird.values[iRow][1] = 'Pending' elif payload.status == Status.Requested: self.ImagesGird.values[iRow][1] = 'Requested' self.display() def on_ok(self): notify_result = npyscreen.notify_ok_cancel("Are you sure you want to exit ?", title='Exit ?',form_color='CONTROL',wrap=True,editw = 1) if(notify_result): npyscreen.notify("Cleaning up before Exiting", title='Exiting') for payload,iRow in self.parentApp.payloads.items(): try: payload.downloader.stop() except Exception: pass self.thread_stop = True while self.checker_requester_thread.is_alive(): pass sys.exit() def PopulateGrid(self): currentGridRow = 0 self.ImagesGird.values = [] payloads = self.parentApp.payloads already_proccessed = 0 for key,(payload,irow) in payloads.items(): row = [] if(payload.queued_for_download): npyscreen.notify("Fetching additional data:\nFetched for {} of {} products".format(already_proccessed,self.count_selected), title='Fetching Data',wrap=True) payloads[key][1] = currentGridRow payloads[key][0].configure_downloader() self.rowToKey[currentGridRow] = key row.append(payload.product_short_name) if(payload.status == Status.Online): row.append('Online') else: row.append('Offline') row.append(' (0 MB of {})'.format(utils.sizeof_human(payload.size))) row.append(' ') row.append(' ') currentGridRow += 1 self.ImagesGird.values.append(row) already_proccessed +=1 self.main_timer.tick() self.checker_requester_thread = threading.Thread(name="Request Checker",target=self.checker_requester_fun, args=()) self.checker_requester_thread.isDaemon = True self.checker_requester_thread.name = "Checker Thread" self.checker_requester_thread.start() self.ImagesGird.editable = True def checker_requester_fun(self): cooldown_period = 5 cooldown_timer = Timer() cooldown_timer.tick() while True: if self.thread_stop == True: break # if cooldown_timer.has_elapsed(cooldown_period): # cooldown_timer.tick() for payload,iRow in self.parentApp.payloads.values(): if self.thread_stop: break if not payload.queued_for_download or (payload.status > Status.Downloading and not payload.status == Status.Failed): continue payload.auto_update_status(self.main_timer) time.sleep(10) #define ^p and ^n key presses when in console for consistency with the previous TUI def h_console_previous(self,inpt): self.Console.h_exit_up(inpt) def h_console_next(self,inpt): self.Console.h_exit_down(inpt) if __name__ == "__main__": os.system('mode con: cols={} lines={}'.format(140,40)) npyscreen.wrapper(MainTUI().run())