jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improved detection of incomplete or not yet downloaded data by merger_FITS

Browse source
This commit is contained in:
Jeremy Karst 2024-05-29 13:49:28 -04:00
parent e1b764cfa1
commit 57c9aece2c
1 changed files with 58 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
merger_FITS.py
View file

@ -1,5 +1,6 @@
import os
import time
import calendar
import datetime
from collections import defaultdict
from multiprocessing import Queue, Process
@ -434,15 +435,10 @@ image_names = ["094A", "131A", "171A", "195A", "284A", "304A"]
if __name__ == "__main__":
# stored_fits_dirs = [r"..\Data\goes16\l2\data", r"..\Data\goes18\l2\data"]
# processed_images_dirs = [r"..\composite\goes16", r"..\composite\goes18"]
stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
processed_images_dir = r"..\composite\goes18"
starttime = time.mktime(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = time.mktime(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stored_fits_dirs = [r"..\Data\goes16",]
processed_images_dir = r"..\composite\goes16"
starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$"
nworkers = 20
@ -468,7 +464,7 @@ if __name__ == "__main__":
black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
try:
files_sorted_by_timestamp = defaultdict(list)
found_files = 0
num_found_files = 0
for stored_fits_dir in stored_fits_dirs:
os.makedirs(processed_images_dir, exist_ok=True)
filename_tester = re.compile(regex_filename)
@ -480,10 +476,13 @@ if __name__ == "__main__":
# measurement = file_parts[1]
# sattelite = file_parts[2]
measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
if (measure_end_time >= starttime) and (measure_end_time < stoptime):
files_sorted_by_timestamp[measure_end_time].append(os.path.join(root,f))
found_files += 1
num_found_files += 1
print(f"Found {found_files} FITS files. Starting conversion.")
print(f"Found {num_found_files} FITS files. Starting conversion.")
if num_found_files == 0:
exit(3)
sorted_times = sorted(list(files_sorted_by_timestamp.keys()))
min_time = sorted_times[0]
@ -495,6 +494,24 @@ if __name__ == "__main__":
last_good_files = None
last_good_file_times = None
# Lets find the first and last timestamps in the sorted_times from our files which actually have a full set of 6/6 images available
# This check will prevent partially downloaded sets of data from generating composite images which have "filled in" data from detected gaps
# which would have been later filled with downloaded imagery.
f = None
l = None
for i, timestamp in enumerate(sorted_times):
if len(files_sorted_by_timestamp[timestamp]) == 6:
f = i
break
for i, timestamp in enumerate(reversed(sorted_times)):
if len(files_sorted_by_timestamp[timestamp]) == 6:
l = len(sorted_times) - 1 - i
break
assert f is not None # Check to make sure we found valid indices
assert l is not None
assert f != l
sorted_times = sorted_times[f:l] # Limit our composite image generation to only files within the valid range
last_good_files = files_sorted_by_timestamp[sorted_times[0]]
for timestamp in tqdm.tqdm(sorted_times, desc="Creating Composite Solar Images"):
if timestamp < min_time or timestamp > max_time:
continue
@ -517,10 +534,6 @@ if __name__ == "__main__":
files_this_timestamp = sorted(files_this_timestamp)
synthetic_data = False
if (not len(files_this_timestamp) == 6):
if (not last_good_files):
print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 and no last-good data.")
continue
else:
print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.")
files_for_job = []
for i, prefix in enumerate(file_prefixes):