diff --git a/puller_fits.py b/puller_fits.py index b04b8aa..ec47928 100644 --- a/puller_fits.py +++ b/puller_fits.py @@ -15,7 +15,7 @@ directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-sat stored_images_dir = r"..\Data" # directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/suvi-l2-ci094/" # stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16\l2\suvi-l2-ci094" -ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020"] +ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022"] file_database_path = r"..\file_database.json" fetch_interval = 0 # 60*60 nworkers = 8 @@ -132,23 +132,40 @@ if __name__ == "__main__": failed_image_count += 1 except queue.Empty: break + + file_portion_of_link = l[urllen:] + filepath = os.path.join(stored_images_dir, file_portion_of_link.replace("/", os.sep)) # If we dont have the file or the file at the link is newer than the one we previously fetched if (not (l in file_info_cache)) or t > file_info_cache[l]: - file_portion_of_link = l[urllen:] - filepath = os.path.join(stored_images_dir, file_portion_of_link.replace("/", os.sep)) + if filepath.endswith(".fits"): + filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file + if os.path.exists(filepath2): # If the unfiltered filename exists, that case will be handled in the alternative code path in if os.path.exists(filepath): + if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version. + os.remove(filepath2) + else: # If we have no record of this file, update the file info cache and don't redownload + file_info_cache[l] = t + already_had_image_count += 1 + continue if os.path.exists(filepath): - if l in file_info_cache: # If we have record of this file, it must be out of date, rename it and download the new version. - new_name = filepath + f"_{int(file_info_cache[l])}" - if os.path.exists(new_name): - os.remove(new_name) - os.rename(filepath, new_name) + if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version. + if os.path.exists(filepath): + os.remove(filepath) else: # If we have no record of this file, update the file info cache and don't redownload file_info_cache[l] = t already_had_image_count += 1 continue work_queue.put((l, filepath, t)) else: - already_had_image_count += 1 + # We have a download record, confirm the file actually exists on disk + if os.path.exists(filepath): + already_had_image_count += 1 + elif filepath.endswith(".fits"): + filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file + if os.path.exists(filepath2): + already_had_image_count += 1 + else: # We could not find the file on disk, queue for redownload + work_queue.put((l, filepath, t)) + with open(file_database_path, 'w') as f: f.write(json.dumps(file_info_cache)) print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")