jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cleaned up merger and added flag for generating synthetic data.

Browse source
This commit is contained in:
Jeremy Karst 2024-06-04 17:18:51 -04:00
parent 4471e1c503
commit d5545bc0e5
1 changed files with 84 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

179
merger_FITS.py
View file

@ -433,16 +433,15 @@ def generate_composite(work_queue, result_queue):
image_names = ["094A", "131A", "171A", "195A", "284A", "304A"] image_names = ["094A", "131A", "171A", "195A", "284A", "304A"]
if __name__ == "__main__": if __name__ == "__main__":
# stored_fits_dirs = [r"..\Data\goes16\l2\data", r"..\Data\goes18\l2\data"] stored_fits_dirs = [r"..\Data\goes16\l2\data", r"..\Data\goes18\l2\data"]
# processed_images_dirs = [r"..\composite\goes16", r"..\composite\goes18"] processed_images_dirs = [r"..\composite\goes16", r"..\composite\goes18"]
stored_fits_dirs = [r"..\Data\goes16",] starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
processed_images_dir = r"..\composite\goes16" stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$" regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$"
nworkers = 20 nworkers = 20
max_time_gap = 3 max_time_gap = 3
fill_missing_data = False
file_prefixes = ["dr_suvi-l2-ci" + n[:-1] for n in image_names] file_prefixes = ["dr_suvi-l2-ci" + n[:-1] for n in image_names]
@ -463,9 +462,9 @@ if __name__ == "__main__":
nfailed = 0 nfailed = 0
black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8')) black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
try: try:
files_sorted_by_timestamp = defaultdict(list) for stored_fits_dir, processed_images_dir in zip(stored_fits_dirs, processed_images_dirs):
num_found_files = 0 files_by_timestamp = defaultdict(list)
for stored_fits_dir in stored_fits_dirs: num_found_files = 0
os.makedirs(processed_images_dir, exist_ok=True) os.makedirs(processed_images_dir, exist_ok=True)
filename_tester = re.compile(regex_filename) filename_tester = re.compile(regex_filename)
print(f"Searching for FITS files in: {stored_fits_dir}") print(f"Searching for FITS files in: {stored_fits_dir}")
@ -477,98 +476,88 @@ if __name__ == "__main__":
# sattelite = file_parts[2] # sattelite = file_parts[2]
measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp()) measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
if (measure_end_time >= starttime) and (measure_end_time < stoptime): if (measure_end_time >= starttime) and (measure_end_time < stoptime):
files_sorted_by_timestamp[measure_end_time].append(os.path.join(root,f)) files_by_timestamp[measure_end_time].append(os.path.join(root,f))
num_found_files += 1 num_found_files += 1
print(f"Found {num_found_files} FITS files. Starting conversion.") print(f"Found {num_found_files} FITS files. Starting conversion.")
if num_found_files == 0: if num_found_files == 0:
exit(3) exit(3)
sorted_times = sorted(list(files_sorted_by_timestamp.keys())) sorted_times = sorted(list(files_by_timestamp.keys()))
min_time = sorted_times[0] min_time = sorted_times[0]
max_time = sorted_times[-1] max_time = sorted_times[-1]
diff_times = np.diff(sorted_times) diff_times = np.diff(sorted_times)
unique, counts = np.unique(diff_times, return_counts=True) unique, counts = np.unique(diff_times, return_counts=True)
interval = unique[0] # This is the amount of time between each sample in seconds interval = unique[0] # This is the amount of time between each sample in seconds
assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval
last_good_files = None last_good_files = None
last_good_file_times = None last_good_file_times = None
# Lets find the first and last timestamps in the sorted_times from our files which actually have a full set of 6/6 images available # Lets find the first and last timestamps in the sorted_times from our files which actually have a full set of 6/6 images available
# This check will prevent partially downloaded sets of data from generating composite images which have "filled in" data from detected gaps # This check will prevent partially downloaded sets of data from generating composite images which have "filled in" data from detected gaps
# which would have been later filled with downloaded imagery. # which would have been later filled with downloaded imagery.
f = None f = None
l = None l = None
for i, timestamp in enumerate(sorted_times): for i, timestamp in enumerate(sorted_times):
if len(files_sorted_by_timestamp[timestamp]) == 6: if len(files_by_timestamp[timestamp]) == 6:
f = i f = i
break
for i, timestamp in enumerate(reversed(sorted_times)):
if len(files_sorted_by_timestamp[timestamp]) == 6:
l = len(sorted_times) - 1 - i
break
assert f is not None # Check to make sure we found valid indices
assert l is not None
assert f != l
sorted_times = sorted_times[f:l] # Limit our composite image generation to only files within the valid range
last_good_files = files_sorted_by_timestamp[sorted_times[0]]
for timestamp in tqdm.tqdm(sorted_times, desc="Creating Composite Solar Images"):
if timestamp < min_time or timestamp > max_time:
continue
# Collect completed jobs and record completion status
while True:
try:
result = result_queue.get_nowait()
if result[0] == "Exists":
nexists += 1
elif result[0] == "Created":
ncreated += 1
else:
print(f"A worker encountered an exception on job {result[1]}: {result[0]}")
nfailed += 1
except queue.Empty:
break break
for i, timestamp in enumerate(reversed(sorted_times)):
# Submit new jobs if len(files_by_timestamp[timestamp]) == 6:
files_this_timestamp = files_sorted_by_timestamp[timestamp] l = len(sorted_times) - 1 - i
files_this_timestamp = sorted(files_this_timestamp) break
synthetic_data = False assert f is not None # Check to make sure we found valid indices
if (not len(files_this_timestamp) == 6): assert l is not None
print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.") assert f != l
files_for_job = [] sorted_times = sorted_times[f:l] # Limit our composite image generation to only files within the valid range
for i, prefix in enumerate(file_prefixes): last_good_files = files_by_timestamp[sorted_times[0]]
found = False for timestamp in tqdm.tqdm(sorted_times, desc="Creating Composite Solar Images"):
for f in files_this_timestamp: if timestamp < min_time or timestamp > max_time:
filename = os.path.split(f)[-1] continue
if filename.startswith(prefix): # Collect completed jobs and record completion status
files_for_job.append(f) while True:
last_good_files[i] = f try:
last_good_file_times[i] = timestamp result = result_queue.get_nowait()
found = True if result[0] == "Exists":
break nexists += 1
if not found: # We did not find this prefix, use the last good file elif result[0] == "Created":
time_gap = (timestamp - last_good_file_times[i]) // interval ncreated += 1
if time_gap <= max_time_gap:
synthetic_data = True
files_for_job.append(last_good_files[i])
else: else:
print(f"Detected a gap of {time_gap} frames at {timestamp}, inserting black frames.") print(f"A worker encountered an exception on job {result[1]}: {result[0]}")
filename = f"Composite-{int(timestamp)}_b.jpg" nfailed += 1
filepath = os.path.join(processed_images_dir, filename) except queue.Empty:
if not os.path.isfile(filepath): break
black_image.save(filepath, quality = 95)
break # Submit new jobs
if len(files_for_job) == 6: files_this_timestamp = files_by_timestamp[timestamp]
work_queue.put((files_for_job, timestamp, processed_images_dir, synthetic_data)) files_this_timestamp = sorted(files_this_timestamp)
else: if (not len(files_this_timestamp) == 6):
# We did not get a full file set to process, because we were missing one or more files and also exceeeded time_gap limits if fill_missing_data:
pass print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.")
else: # We have a complete file set, update the last_good_files files_for_job = []
last_good_files = files_this_timestamp for i, prefix in enumerate(file_prefixes):
last_good_file_times = [timestamp for _ in last_good_files] found = False
if len(files_this_timestamp) != 6: for f in files_this_timestamp:
print("!!!!!") filename = os.path.split(f)[-1]
work_queue.put((files_this_timestamp, timestamp, processed_images_dir, synthetic_data)) if filename.startswith(prefix):
files_for_job.append(f)
last_good_files[i] = f
last_good_file_times[i] = timestamp
found = True
break
if not found: # We did not find this prefix, use the last good file
time_gap = (timestamp - last_good_file_times[i]) // interval
if time_gap <= max_time_gap:
files_for_job.append(last_good_files[i])
else:
print(f"Detected a gap of {time_gap} frames at {timestamp}")
break
if len(files_for_job) == 6:
work_queue.put((files_for_job, timestamp, processed_images_dir, True))
else: # We have a complete file set, update the last_good_files
last_good_files = files_this_timestamp
last_good_file_times = [timestamp for _ in last_good_files]
work_queue.put((files_this_timestamp, timestamp, processed_images_dir, False))
except KeyboardInterrupt: except KeyboardInterrupt:
print("Finishing current jobs and exiting") print("Finishing current jobs and exiting")