Improved detection of incomplete or not yet downloaded data by merger_FITS

merger_FITS.py

@@ -1,5 +1,6 @@
 import os
 import time
+import calendar
 import datetime
 from collections import defaultdict
 from multiprocessing import Queue, Process
@@ -434,15 +435,10 @@ image_names = ["094A", "131A", "171A", "195A", "284A", "304A"]
 if __name__ == "__main__":
     # stored_fits_dirs = [r"..\Data\goes16\l2\data", r"..\Data\goes18\l2\data"]
     # processed_images_dirs = [r"..\composite\goes16", r"..\composite\goes18"]
-    stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
+    stored_fits_dirs = [r"..\Data\goes16",]
-                        r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
+    processed_images_dir = r"..\composite\goes16"
-                        r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
+    starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-                        r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
+    stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-                        r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
-                        r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
-    processed_images_dir = r"..\composite\goes18"
-    starttime = time.mktime(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-    stoptime = time.mktime(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
 
     regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$"
     nworkers = 20
@@ -468,7 +464,7 @@ if __name__ == "__main__":
     black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
     try:
         files_sorted_by_timestamp = defaultdict(list)
-        found_files = 0
+        num_found_files = 0
         for stored_fits_dir in stored_fits_dirs:
             os.makedirs(processed_images_dir, exist_ok=True)
             filename_tester = re.compile(regex_filename)
@@ -480,10 +476,13 @@ if __name__ == "__main__":
                         # measurement = file_parts[1]
                         # sattelite = file_parts[2]
                         measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
+                        if (measure_end_time >= starttime) and (measure_end_time < stoptime):
                             files_sorted_by_timestamp[measure_end_time].append(os.path.join(root,f))
-                        found_files += 1
+                            num_found_files += 1
                     
-        print(f"Found {found_files} FITS files. Starting conversion.")
+        print(f"Found {num_found_files} FITS files. Starting conversion.")
+        if num_found_files == 0:
+            exit(3)
 
         sorted_times = sorted(list(files_sorted_by_timestamp.keys()))
         min_time = sorted_times[0]
@@ -495,6 +494,24 @@ if __name__ == "__main__":
 
         last_good_files = None
         last_good_file_times = None
+        # Lets find the first and last timestamps in the sorted_times from our files which actually have a full set of 6/6 images available
+        #    This check will prevent partially downloaded sets of data from generating composite images which have "filled in" data from detected gaps
+        #    which would have been later filled with downloaded imagery.
+        f = None
+        l = None
+        for i, timestamp in enumerate(sorted_times):
+            if len(files_sorted_by_timestamp[timestamp]) == 6:
+                f = i
+                break
+        for i, timestamp in enumerate(reversed(sorted_times)):
+            if len(files_sorted_by_timestamp[timestamp]) == 6:
+                l = len(sorted_times) - 1 - i
+                break
+        assert f is not None # Check to make sure we found valid indices
+        assert l is not None
+        assert f != l 
+        sorted_times = sorted_times[f:l] # Limit our composite image generation to only files within the valid range
+        last_good_files = files_sorted_by_timestamp[sorted_times[0]]
         for timestamp in tqdm.tqdm(sorted_times, desc="Creating Composite Solar Images"):
             if timestamp < min_time or timestamp > max_time:
                 continue
@@ -517,10 +534,6 @@ if __name__ == "__main__":
             files_this_timestamp = sorted(files_this_timestamp)
             synthetic_data = False
             if (not len(files_this_timestamp) == 6):
-                if (not last_good_files):
-                    print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 and no last-good data.")
-                    continue
-                else:
                 print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.")
                 files_for_job = []
                 for i, prefix in enumerate(file_prefixes):