From e1b764cfa1a48c7fdb5dab626703731b8c7904a8 Mon Sep 17 00:00:00 2001 From: Jeremy Karst Date: Wed, 29 May 2024 13:48:59 -0400 Subject: [PATCH] Added error file handling to puller_fits --- puller_fits.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/puller_fits.py b/puller_fits.py index ec47928..a05c239 100644 --- a/puller_fits.py +++ b/puller_fits.py @@ -15,7 +15,7 @@ directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-sat stored_images_dir = r"..\Data" # directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/suvi-l2-ci094/" # stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16\l2\suvi-l2-ci094" -ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022"] +ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"] file_database_path = r"..\file_database.json" fetch_interval = 0 # 60*60 nworkers = 8 @@ -139,9 +139,14 @@ if __name__ == "__main__": if (not (l in file_info_cache)) or t > file_info_cache[l]: if filepath.endswith(".fits"): filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file - if os.path.exists(filepath2): # If the unfiltered filename exists, that case will be handled in the alternative code path in if os.path.exists(filepath): + filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file + if os.path.exists(filepath2) or os.path.exists(filepath3): # If the unfiltered filename exists, that case will be handled in the alternative code path in if os.path.exists(filepath): if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version. - os.remove(filepath2) + try: + os.remove(filepath2) + os.remove(filepath3) + except: + pass else: # If we have no record of this file, update the file info cache and don't redownload file_info_cache[l] = t already_had_image_count += 1 @@ -161,7 +166,8 @@ if __name__ == "__main__": already_had_image_count += 1 elif filepath.endswith(".fits"): filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file - if os.path.exists(filepath2): + filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file + if os.path.exists(filepath2) or os.path.exists(filepath3): already_had_image_count += 1 else: # We could not find the file on disk, queue for redownload work_queue.put((l, filepath, t))