Improved detection of incomplete or not yet downloaded data by merger_FITS

merger_FITS.py

@@ -1,5 +1,6 @@
 import os
 import time
+import calendar
 import datetime
 from collections import defaultdict
 from multiprocessing import Queue, Process
@@ -434,15 +435,10 @@ image_names = ["094A", "131A", "171A", "195A", "284A", "304A"]
 if __name__ == "__main__":
     # stored_fits_dirs = [r"..\Data\goes16\l2\data", r"..\Data\goes18\l2\data"]
     # processed_images_dirs = [r"..\composite\goes16", r"..\composite\goes18"]
-    stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
+    stored_fits_dirs = [r"..\Data\goes16",]
-                        r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
+    processed_images_dir = r"..\composite\goes16"
-                        r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
+    starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-                        r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
+    stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-                        r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
-                        r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
-    processed_images_dir = r"..\composite\goes18"
-    starttime = time.mktime(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-    stoptime = time.mktime(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
 
     regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$"
     nworkers = 20
@@ -468,7 +464,7 @@ if __name__ == "__main__":
     black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
     try:
         files_sorted_by_timestamp = defaultdict(list)
-        found_files = 0
+        num_found_files = 0
         for stored_fits_dir in stored_fits_dirs:
             os.makedirs(processed_images_dir, exist_ok=True)
             filename_tester = re.compile(regex_filename)
@@ -480,10 +476,13 @@ if __name__ == "__main__":
                         # measurement = file_parts[1]
                         # sattelite = file_parts[2]
                         measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
-                        files_sorted_by_timestamp[measure_end_time].append(os.path.join(root,f))
+                        if (measure_end_time >= starttime) and (measure_end_time < stoptime):
-                        found_files += 1
+                            files_sorted_by_timestamp[measure_end_time].append(os.path.join(root,f))
+                            num_found_files += 1
                     
-        print(f"Found {found_files} FITS files. Starting conversion.")
+        print(f"Found {num_found_files} FITS files. Starting conversion.")
+        if num_found_files == 0:
+            exit(3)
 
         sorted_times = sorted(list(files_sorted_by_timestamp.keys()))
         min_time = sorted_times[0]
@@ -495,6 +494,24 @@ if __name__ == "__main__":
 
         last_good_files = None
         last_good_file_times = None
+        # Lets find the first and last timestamps in the sorted_times from our files which actually have a full set of 6/6 images available
+        #    This check will prevent partially downloaded sets of data from generating composite images which have "filled in" data from detected gaps
+        #    which would have been later filled with downloaded imagery.
+        f = None
+        l = None
+        for i, timestamp in enumerate(sorted_times):
+            if len(files_sorted_by_timestamp[timestamp]) == 6:
+                f = i
+                break
+        for i, timestamp in enumerate(reversed(sorted_times)):
+            if len(files_sorted_by_timestamp[timestamp]) == 6:
+                l = len(sorted_times) - 1 - i
+                break
+        assert f is not None # Check to make sure we found valid indices
+        assert l is not None
+        assert f != l 
+        sorted_times = sorted_times[f:l] # Limit our composite image generation to only files within the valid range
+        last_good_files = files_sorted_by_timestamp[sorted_times[0]]
         for timestamp in tqdm.tqdm(sorted_times, desc="Creating Composite Solar Images"):
             if timestamp < min_time or timestamp > max_time:
                 continue
@@ -517,39 +534,35 @@ if __name__ == "__main__":
             files_this_timestamp = sorted(files_this_timestamp)
             synthetic_data = False
             if (not len(files_this_timestamp) == 6):
-                if (not last_good_files):
+                print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.")
-                    print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 and no last-good data.")
+                files_for_job = []
-                    continue
+                for i, prefix in enumerate(file_prefixes):
+                    found = False
+                    for f in files_this_timestamp:
+                        filename = os.path.split(f)[-1]
+                        if filename.startswith(prefix):
+                            files_for_job.append(f)
+                            last_good_files[i] = f
+                            last_good_file_times[i] = timestamp
+                            found = True
+                            break
+                    if not found: # We did not find this prefix, use the last good file
+                        time_gap = (timestamp - last_good_file_times[i]) // interval
+                        if time_gap <= max_time_gap:
+                            synthetic_data = True
+                            files_for_job.append(last_good_files[i])
+                        else:
+                            print(f"Detected a gap of {time_gap} frames at {timestamp}, inserting black frames.")
+                            filename = f"Composite-{int(timestamp)}_b.jpg"
+                            filepath = os.path.join(processed_images_dir, filename)
+                            if not os.path.isfile(filepath):
+                                black_image.save(filepath, quality = 95)
+                            break
+                if len(files_for_job) == 6:
+                    work_queue.put((files_for_job, timestamp, processed_images_dir, synthetic_data))
                 else:
-                    print(f"Invalid or incomplete sensor records for {timestamp} - {len(files_this_timestamp)}/6 filling from last good data.")
+                    # We did not get a full file set to process, because we were missing one or more files and also exceeeded time_gap limits
-                    files_for_job = []
+                    pass
-                    for i, prefix in enumerate(file_prefixes):
-                        found = False
-                        for f in files_this_timestamp:
-                            filename = os.path.split(f)[-1]
-                            if filename.startswith(prefix):
-                                files_for_job.append(f)
-                                last_good_files[i] = f
-                                last_good_file_times[i] = timestamp
-                                found = True
-                                break
-                        if not found: # We did not find this prefix, use the last good file
-                            time_gap = (timestamp - last_good_file_times[i]) // interval
-                            if time_gap <= max_time_gap:
-                                synthetic_data = True
-                                files_for_job.append(last_good_files[i])
-                            else:
-                                print(f"Detected a gap of {time_gap} frames at {timestamp}, inserting black frames.")
-                                filename = f"Composite-{int(timestamp)}_b.jpg"
-                                filepath = os.path.join(processed_images_dir, filename)
-                                if not os.path.isfile(filepath):
-                                    black_image.save(filepath, quality = 95)
-                                break
-                    if len(files_for_job) == 6:
-                        work_queue.put((files_for_job, timestamp, processed_images_dir, synthetic_data))
-                    else:
-                        # We did not get a full file set to process, because we were missing one or more files and also exceeeded time_gap limits
-                        pass
             else: # We have a complete file set, update the last_good_files
                 last_good_files = files_this_timestamp
                 last_good_file_times = [timestamp for _ in last_good_files]