diff --git a/merger_FITS.py b/merger_FITS.py index 638f724..fc703f8 100644 --- a/merger_FITS.py +++ b/merger_FITS.py @@ -1,5 +1,6 @@ import os import time +import calendar import datetime from collections import defaultdict from multiprocessing import Queue, Process @@ -434,15 +435,10 @@ image_names = ["094A", "131A", "171A", "195A", "284A", "304A"] if __name__ == "__main__": # stored_fits_dirs = [r"..\Data\goes16\l2\data", r"..\Data\goes18\l2\data"] # processed_images_dirs = [r"..\composite\goes16", r"..\composite\goes18"] - stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci131\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci171\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci195\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci284\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",] - processed_images_dir = r"..\composite\goes18" - starttime = time.mktime(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) - stoptime = time.mktime(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) + stored_fits_dirs = [r"..\Data\goes16",] + processed_images_dir = r"..\composite\goes16" + starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) + stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$" nworkers = 20 @@ -468,7 +464,7 @@ if __name__ == "__main__": black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8')) try: files_sorted_by_timestamp = defaultdict(list) - found_files = 0 + num_found_files = 0 for stored_fits_dir in stored_fits_dirs: os.makedirs(processed_images_dir, exist_ok=True) filename_tester = re.compile(regex_filename) @@ -480,10 +476,13 @@ if __name__ == "__main__": # measurement = file_parts[1] # sattelite = file_parts[2] measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp()) - files_sorted_by_timestamp[measure_end_time].append(os.path.join(root,f)) - found_files += 1 + if (measure_end_time >= starttime) and (measure_end_time < stoptime): + files_sorted_by_timestamp[measure_end_time].append(os.path.join(root,f)) + num_found_files += 1 - print(f"Found {found_files} FITS files. Starting conversion.") + print(f"Found {num_found_files} FITS files. Starting conversion.") + if num_found_files == 0: + exit(3) sorted_times = sorted(list(files_sorted_by_timestamp.keys())) min_time = sorted_times[0] @@ -495,6 +494,24 @@ if __name__ == "__main__": last_good_files = None last_good_file_times = None + # Lets find the first and last timestamps in the sorted_times from our files which actually have a full set of 6/6 images available + # This check will prevent partially downloaded sets of data from generating composite images which have "filled in" data from detected gaps + # which would have been later filled with downloaded imagery. + f = None + l = None + for i, timestamp in enumerate(sorted_times): + if len(files_sorted_by_timestamp[timestamp]) == 6: + f = i + break + for i, timestamp in enumerate(reversed(sorted_times)): + if len(files_sorted_by_timestamp[timestamp]) == 6: + l = len(sorted_times) - 1 - i + break + assert f is not None # Check to make sure we found valid indices + assert l is not None + assert f != l + sorted_times = sorted_times[f:l] # Limit our composite image generation to only files within the valid range + last_good_files = files_sorted_by_timestamp[sorted_times[0]] for timestamp in tqdm.tqdm(sorted_times, desc="Creating Composite Solar Images"): if timestamp < min_time or timestamp > max_time: continue @@ -517,39 +534,35 @@ if __name__ == "__main__": files_this_timestamp = sorted(files_this_timestamp) synthetic_data = False if (not len(files_this_timestamp) == 6): - if (not last_good_files): - print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 and no last-good data.") - continue + print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.") + files_for_job = [] + for i, prefix in enumerate(file_prefixes): + found = False + for f in files_this_timestamp: + filename = os.path.split(f)[-1] + if filename.startswith(prefix): + files_for_job.append(f) + last_good_files[i] = f + last_good_file_times[i] = timestamp + found = True + break + if not found: # We did not find this prefix, use the last good file + time_gap = (timestamp - last_good_file_times[i]) // interval + if time_gap <= max_time_gap: + synthetic_data = True + files_for_job.append(last_good_files[i]) + else: + print(f"Detected a gap of {time_gap} frames at {timestamp}, inserting black frames.") + filename = f"Composite-{int(timestamp)}_b.jpg" + filepath = os.path.join(processed_images_dir, filename) + if not os.path.isfile(filepath): + black_image.save(filepath, quality = 95) + break + if len(files_for_job) == 6: + work_queue.put((files_for_job, timestamp, processed_images_dir, synthetic_data)) else: - print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.") - files_for_job = [] - for i, prefix in enumerate(file_prefixes): - found = False - for f in files_this_timestamp: - filename = os.path.split(f)[-1] - if filename.startswith(prefix): - files_for_job.append(f) - last_good_files[i] = f - last_good_file_times[i] = timestamp - found = True - break - if not found: # We did not find this prefix, use the last good file - time_gap = (timestamp - last_good_file_times[i]) // interval - if time_gap <= max_time_gap: - synthetic_data = True - files_for_job.append(last_good_files[i]) - else: - print(f"Detected a gap of {time_gap} frames at {timestamp}, inserting black frames.") - filename = f"Composite-{int(timestamp)}_b.jpg" - filepath = os.path.join(processed_images_dir, filename) - if not os.path.isfile(filepath): - black_image.save(filepath, quality = 95) - break - if len(files_for_job) == 6: - work_queue.put((files_for_job, timestamp, processed_images_dir, synthetic_data)) - else: - # We did not get a full file set to process, because we were missing one or more files and also exceeeded time_gap limits - pass + # We did not get a full file set to process, because we were missing one or more files and also exceeeded time_gap limits + pass else: # We have a complete file set, update the last_good_files last_good_files = files_this_timestamp last_good_file_times = [timestamp for _ in last_good_files]