jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added filtered and error suffixes to puller

Browse source
This commit is contained in:
Jeremy Karst 2024-05-20 19:44:01 -04:00
parent 133a4f563c
commit a77bee2b7e
1 changed files with 26 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
puller_fits.py
View file

@ -15,7 +15,7 @@ directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-sat
stored_images_dir = r"..\Data" stored_images_dir = r"..\Data"
# directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/suvi-l2-ci094/" # directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/suvi-l2-ci094/"
# stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16\l2\suvi-l2-ci094" # stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16\l2\suvi-l2-ci094"
ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020"] ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022"]
file_database_path = r"..\file_database.json" file_database_path = r"..\file_database.json"
fetch_interval = 0 # 60*60 fetch_interval = 0 # 60*60
nworkers = 8 nworkers = 8
@ -132,23 +132,40 @@ if __name__ == "__main__":
failed_image_count += 1 failed_image_count += 1
except queue.Empty: except queue.Empty:
break break
file_portion_of_link = l[urllen:]
filepath = os.path.join(stored_images_dir, file_portion_of_link.replace("/", os.sep))
# If we dont have the file or the file at the link is newer than the one we previously fetched # If we dont have the file or the file at the link is newer than the one we previously fetched
if (not (l in file_info_cache)) or t > file_info_cache[l]: if (not (l in file_info_cache)) or t > file_info_cache[l]:
file_portion_of_link = l[urllen:] if filepath.endswith(".fits"):
filepath = os.path.join(stored_images_dir, file_portion_of_link.replace("/", os.sep)) filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file
if os.path.exists(filepath2): # If the unfiltered filename exists, that case will be handled in the alternative code path in if os.path.exists(filepath):
if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
os.remove(filepath2)
else: # If we have no record of this file, update the file info cache and don't redownload
file_info_cache[l] = t
already_had_image_count += 1
continue
if os.path.exists(filepath): if os.path.exists(filepath):
if l in file_info_cache: # If we have record of this file, it must be out of date, rename it and download the new version. if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
new_name = filepath + f"_{int(file_info_cache[l])}" if os.path.exists(filepath):
if os.path.exists(new_name): os.remove(filepath)
os.remove(new_name)
os.rename(filepath, new_name)
else: # If we have no record of this file, update the file info cache and don't redownload else: # If we have no record of this file, update the file info cache and don't redownload
file_info_cache[l] = t file_info_cache[l] = t
already_had_image_count += 1 already_had_image_count += 1
continue continue
work_queue.put((l, filepath, t)) work_queue.put((l, filepath, t))
else: else:
already_had_image_count += 1 # We have a download record, confirm the file actually exists on disk
if os.path.exists(filepath):
already_had_image_count += 1
elif filepath.endswith(".fits"):
filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file
if os.path.exists(filepath2):
already_had_image_count += 1
else: # We could not find the file on disk, queue for redownload
work_queue.put((l, filepath, t))
with open(file_database_path, 'w') as f: with open(file_database_path, 'w') as f:
f.write(json.dumps(file_info_cache)) f.write(json.dumps(file_info_cache))
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}") print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")