jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cleaned up merger and added flag for generating synthetic data.

Browse source
This commit is contained in:
Jeremy Karst 2024-06-04 17:18:51 -04:00
parent 4471e1c503
commit d5545bc0e5
1 changed files with 84 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
merger_FITS.py
View file

@ -433,16 +433,15 @@ def generate_composite(work_queue, result_queue):
image_names = ["094A", "131A", "171A", "195A", "284A", "304A"] image_names = ["094A", "131A", "171A", "195A", "284A", "304A"]
if __name__ == "__main__": if __name__ == "__main__":
# stored_fits_dirs = [r"..\Data\goes16\l2\data", r"..\Data\goes18\l2\data"] stored_fits_dirs = [r"..\Data\goes16\l2\data", r"..\Data\goes18\l2\data"]
# processed_images_dirs = [r"..\composite\goes16", r"..\composite\goes18"] processed_images_dirs = [r"..\composite\goes16", r"..\composite\goes18"]
stored_fits_dirs = [r"..\Data\goes16",] starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
processed_images_dir = r"..\composite\goes16" stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$" regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$"
nworkers = 20 nworkers = 20
max_time_gap = 3 max_time_gap = 3
fill_missing_data = False
file_prefixes = ["dr_suvi-l2-ci" + n[:-1] for n in image_names] file_prefixes = ["dr_suvi-l2-ci" + n[:-1] for n in image_names]
@ -463,9 +462,9 @@ if __name__ == "__main__":
nfailed = 0 nfailed = 0
black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8')) black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
try: try:
files_sorted_by_timestamp = defaultdict(list) for stored_fits_dir, processed_images_dir in zip(stored_fits_dirs, processed_images_dirs):
files_by_timestamp = defaultdict(list)
num_found_files = 0 num_found_files = 0
for stored_fits_dir in stored_fits_dirs:
os.makedirs(processed_images_dir, exist_ok=True) os.makedirs(processed_images_dir, exist_ok=True)
filename_tester = re.compile(regex_filename) filename_tester = re.compile(regex_filename)
print(f"Searching for FITS files in: {stored_fits_dir}") print(f"Searching for FITS files in: {stored_fits_dir}")
@ -477,14 +476,14 @@ if __name__ == "__main__":
# sattelite = file_parts[2] # sattelite = file_parts[2]
measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp()) measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
if (measure_end_time >= starttime) and (measure_end_time < stoptime): if (measure_end_time >= starttime) and (measure_end_time < stoptime):
files_sorted_by_timestamp[measure_end_time].append(os.path.join(root,f)) files_by_timestamp[measure_end_time].append(os.path.join(root,f))
num_found_files += 1 num_found_files += 1
print(f"Found {num_found_files} FITS files. Starting conversion.") print(f"Found {num_found_files} FITS files. Starting conversion.")
if num_found_files == 0: if num_found_files == 0:
exit(3) exit(3)
sorted_times = sorted(list(files_sorted_by_timestamp.keys())) sorted_times = sorted(list(files_by_timestamp.keys()))
min_time = sorted_times[0] min_time = sorted_times[0]
max_time = sorted_times[-1] max_time = sorted_times[-1]
diff_times = np.diff(sorted_times) diff_times = np.diff(sorted_times)
@ -500,18 +499,18 @@ if __name__ == "__main__":
f = None f = None
l = None l = None
for i, timestamp in enumerate(sorted_times): for i, timestamp in enumerate(sorted_times):
if len(files_sorted_by_timestamp[timestamp]) == 6: if len(files_by_timestamp[timestamp]) == 6:
f = i f = i
break break
for i, timestamp in enumerate(reversed(sorted_times)): for i, timestamp in enumerate(reversed(sorted_times)):
if len(files_sorted_by_timestamp[timestamp]) == 6: if len(files_by_timestamp[timestamp]) == 6:
l = len(sorted_times) - 1 - i l = len(sorted_times) - 1 - i
break break
assert f is not None # Check to make sure we found valid indices assert f is not None # Check to make sure we found valid indices
assert l is not None assert l is not None
assert f != l assert f != l
sorted_times = sorted_times[f:l] # Limit our composite image generation to only files within the valid range sorted_times = sorted_times[f:l] # Limit our composite image generation to only files within the valid range
last_good_files = files_sorted_by_timestamp[sorted_times[0]] last_good_files = files_by_timestamp[sorted_times[0]]
for timestamp in tqdm.tqdm(sorted_times, desc="Creating Composite Solar Images"): for timestamp in tqdm.tqdm(sorted_times, desc="Creating Composite Solar Images"):
if timestamp < min_time or timestamp > max_time: if timestamp < min_time or timestamp > max_time:
continue continue
@ -530,10 +529,10 @@ if __name__ == "__main__":
break break
# Submit new jobs # Submit new jobs
files_this_timestamp = files_sorted_by_timestamp[timestamp] files_this_timestamp = files_by_timestamp[timestamp]
files_this_timestamp = sorted(files_this_timestamp) files_this_timestamp = sorted(files_this_timestamp)
synthetic_data = False
if (not len(files_this_timestamp) == 6): if (not len(files_this_timestamp) == 6):
if fill_missing_data:
print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.") print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.")
files_for_job = [] files_for_job = []
for i, prefix in enumerate(file_prefixes): for i, prefix in enumerate(file_prefixes):
@ -549,26 +548,16 @@ if __name__ == "__main__":
if not found: # We did not find this prefix, use the last good file if not found: # We did not find this prefix, use the last good file
time_gap = (timestamp - last_good_file_times[i]) // interval time_gap = (timestamp - last_good_file_times[i]) // interval
if time_gap <= max_time_gap: if time_gap <= max_time_gap:
synthetic_data = True
files_for_job.append(last_good_files[i]) files_for_job.append(last_good_files[i])
else: else:
print(f"Detected a gap of {time_gap} frames at {timestamp}, inserting black frames.") print(f"Detected a gap of {time_gap} frames at {timestamp}")
filename = f"Composite-{int(timestamp)}_b.jpg"
filepath = os.path.join(processed_images_dir, filename)
if not os.path.isfile(filepath):
black_image.save(filepath, quality = 95)
break break
if len(files_for_job) == 6: if len(files_for_job) == 6:
work_queue.put((files_for_job, timestamp, processed_images_dir, synthetic_data)) work_queue.put((files_for_job, timestamp, processed_images_dir, True))
else:
# We did not get a full file set to process, because we were missing one or more files and also exceeeded time_gap limits
pass
else: # We have a complete file set, update the last_good_files else: # We have a complete file set, update the last_good_files
last_good_files = files_this_timestamp last_good_files = files_this_timestamp
last_good_file_times = [timestamp for _ in last_good_files] last_good_file_times = [timestamp for _ in last_good_files]
if len(files_this_timestamp) != 6: work_queue.put((files_this_timestamp, timestamp, processed_images_dir, False))
print("!!!!!")
work_queue.put((files_this_timestamp, timestamp, processed_images_dir, synthetic_data))
except KeyboardInterrupt: except KeyboardInterrupt:
print("Finishing current jobs and exiting") print("Finishing current jobs and exiting")