From 5dcf714bf724de8d9b1fc767ee782d3bb470b9d3 Mon Sep 17 00:00:00 2001
From: Jeremy Karst <Jeremy@karsttech.com>
Date: Tue, 4 Jun 2024 23:44:19 -0400
Subject: [PATCH] Fixed a bug in file queing for filter

---
 filter_FITS.py | 39 ++++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/filter_FITS.py b/filter_FITS.py
index 7b7eec2..74f85f3 100644
--- a/filter_FITS.py
+++ b/filter_FITS.py
@@ -42,7 +42,7 @@ def lowpriority():
         os.nice(1)
 
 measurement_names = ["094", "131", "171", "195", "284", "304"]
-thresholds =        [0.050, 0.10 ,  1.00,  1.40,  0.90,  2.50]
+thresholds =        [0.050, 0.10 ,  1.00,  1.40,  0.93,  2.50]
 max_center_skew = 7
 circle_hough_thresh = 0.75
 expected_dims = 1280
@@ -81,7 +81,7 @@ def filter_fits(work_queue):
             hough_radii = np.arange(383, 393, 1)
             hough_res = hough_circle(filtered_data, hough_radii)
 
-            vals, cxs, cys, rads = hough_circle_peaks(hough_res, hough_radii, threshold=circle_hough_thresh, total_num_peaks=10)
+            vals, cxs, cys, rads = hough_circle_peaks(hough_res, hough_radii, threshold=circle_hough_thresh, total_num_peaks=20)
             found_circle = None
             for v, cx, cy, rad in zip(vals, cxs, cys, rads):
                 xskew = abs(639 - cx)
@@ -90,6 +90,7 @@ def filter_fits(work_queue):
                     found_circle = (cx, cy, rad)
                     break
 
+            # print(f"Plotting file: {job}")
             # plt.figure(f"Data {measurement}")
             # if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1)
             # plt.imshow(data, cmap='jet')
@@ -108,6 +109,7 @@ def filter_fits(work_queue):
             # We have validated this file, rename it appropriately
             new_name = job.split(".fits")[0] + "_f.fits"
             os.rename(job, new_name)
+            # print(f"Renamed: {job} -> {new_name}")
 
 
         except KeyboardInterrupt:
@@ -130,7 +132,8 @@ if __name__ == "__main__":
                         r"..\Data\goes18\l2\data\suvi-l2-ci195\2023",
                         r"..\Data\goes18\l2\data\suvi-l2-ci284\2023",
                         r"..\Data\goes18\l2\data\suvi-l2-ci304\2023",]
-    # stored_fits_dirs = [r"..\Data\goes16\l2\data\suvi-l2-ci131\2023\06\24"]
+    
+    # stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci284\2023\01\04"]
 
     starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
     stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
@@ -164,21 +167,23 @@ if __name__ == "__main__":
                         file_name_end = file_parts[-1].split(".")[0]
                         measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
                         if (measure_end_time >= starttime) and (measure_end_time < stoptime):
-                            files_by_timestamp[measure_end_time].append(os.path.join(root,f))
-                        if file_name_end == "f":
-                            continue # Already filtered from a previous run
-                        elif file_name_end == "e":
-                            if reprocess_errors:
-                                new_file_name = "_".join(file_parts[:-1]) + ".fits"
-                                os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
-                                files_by_timestamp[measure_end_time].append(os.path.abspath(os.path.join(root, new_file_name)))
+                            if file_name_end == "f":
+                                if file_parts[-2] == "f": # We have an accidentally double filtered file...
+                                    new_file_name = "_".join(file_parts[:-1]) + ".fits"
+                                    os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
+                                continue # Already filtered from a previous run
+                            elif file_name_end == "e":
+                                if reprocess_errors:
+                                    new_file_name = "_".join(file_parts[:-1]) + ".fits"
+                                    os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
+                                    files_by_timestamp[measure_end_time].append(os.path.join(root, new_file_name))
+                                else:
+                                    continue
+                            elif file_name_end.startswith("v1-0-"): # This is the normal case for unprocessed data
+                                files_by_timestamp[measure_end_time].append(os.path.join(root,f))
                             else:
-                                continue
-                        elif file_name_end == "v1-0-2": # This is the normal case for unprocessed data
-                            files_by_timestamp[measure_end_time].append(os.path.abspath(os.path.join(root,f)))
-                        else:
-                            # print(f"Error - Unexpected FITS file name: {f}")
-                            pass
+                                # print(f"Error - Unexpected FITS file name: {f}")
+                                pass
         sorted_times = sorted(list(files_by_timestamp.keys()))
         for st in tqdm.tqdm(sorted_times, desc="Filtering files"):
             for f in files_by_timestamp[st]: