From 0241545d5e33bd33d690af6858dab41128a5ce67 Mon Sep 17 00:00:00 2001 From: Jeremy Karst Date: Fri, 24 May 2024 21:23:37 -0400 Subject: [PATCH] Added error metadata to filtered filenames --- filter_FITS.py | 80 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 27 deletions(-) diff --git a/filter_FITS.py b/filter_FITS.py index 6211b61..8de2ca5 100644 --- a/filter_FITS.py +++ b/filter_FITS.py @@ -2,6 +2,7 @@ import os from collections import defaultdict from multiprocessing import Queue, Process import re +import traceback import tqdm import numpy as np @@ -40,7 +41,7 @@ def lowpriority(): measurement_names = ["094", "131", "171", "195", "284", "304"] thresholds = [0.050, 0.10 , 1.00, 1.40, 0.90, 2.50] -max_center_skew = 5 +max_center_skew = 7 circle_hough_thresh = 0.7 expected_dims = 1280 ratio_above_thresh_max = 0.5 @@ -52,7 +53,7 @@ def filter_fits(work_queue): job = work_queue.get() if job is None: return - measurement = job.split("_")[1].split("-")[-1][2:] + measurement = job.split("dr_suvi-l2-")[1].split("_")[0][2:] idx = measurement_names.index(measurement) data = fits.getdata(job) assert data.shape[0] == expected_dims @@ -64,6 +65,15 @@ def filter_fits(work_queue): if ratio_above_thresh > ratio_above_thresh_max: print(f"Exceeded ratio_above_thresh_max, possible data corruption in file: {job}") + new_name = job.split(".fits")[0] + "_e.fits" + os.rename(job, new_name) + + # plt.figure(f"Data {measurement}") + # plt.imshow(data, cmap='jet') + # plt.figure(f"Circle") + # plt.imshow(filtered_data, cmap='jet') + # plt.show() + continue hough_radii = np.arange(387, 394, 1) @@ -77,16 +87,20 @@ def filter_fits(work_queue): if xskew < max_center_skew and yskew < max_center_skew: found_circle = (cx, cy, rad) break - + if not found_circle: print(f"Could not find valid solar disc in file: {job}") - # plt.figure(f"Data {measurement}") - # if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1) - # plt.imshow(data, cmap='jet') - # plt.figure(f"Circle") - # if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1) - # plt.imshow(filtered_data, cmap='jet') - # plt.show() + + plt.figure(f"Data {measurement}") + if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1) + plt.imshow(data, cmap='jet') + plt.figure(f"Circle") + if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1) + plt.imshow(filtered_data, cmap='jet') + plt.show() + + new_name = job.split(".fits")[0] + "_e.fits" + os.rename(job, new_name) continue # We have validated this file, rename it appropriately @@ -98,20 +112,25 @@ def filter_fits(work_queue): return except Exception as e: print(f"Error on file: {job} - {e}") + traceback.print_exception(e) if __name__ == "__main__": - # stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024", - # r"..\Data\goes18\l2\data\suvi-l2-ci131\2024", - # r"..\Data\goes18\l2\data\suvi-l2-ci171\2024", - # r"..\Data\goes18\l2\data\suvi-l2-ci195\2024", - # r"..\Data\goes18\l2\data\suvi-l2-ci284\2024", - # r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",] - stored_fits_dirs = [r"Z:\NOAA GOES Data\Data\goes18\l2\data\suvi-l2-ci094\2024"] + stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024", + r"..\Data\goes18\l2\data\suvi-l2-ci131\2024", + r"..\Data\goes18\l2\data\suvi-l2-ci171\2024", + r"..\Data\goes18\l2\data\suvi-l2-ci195\2024", + r"..\Data\goes18\l2\data\suvi-l2-ci284\2024", + r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",] # stored_fits_dirs = [r"..\Data\goes16\l2\data"] + # stored_fits_dirs = [r"Z:\NOAA GOES Data\fits_test_2024"] + + reprocess_errors = False + nworkers = 20 + regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$" - nworkers = 20 + work_queue = Queue(maxsize = nworkers) workers = [] @@ -134,22 +153,29 @@ if __name__ == "__main__": if filename_tester.match(f): file_parts = f.split("_") file_name_end = file_parts[-1].split(".")[0] - if file_name_end == "e": - # Error file, already processed - continue - elif file_name_end == "f": - if file_parts[-2] == "f": - new_file_name = "_".join(file_parts[:-2]) + "_f.fits" - print("bad file rename, fixing") - os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) + if file_name_end == "f": + # if file_parts[-2] == "f": + # new_file_name = "_".join(file_parts[:-2]) + "_f.fits" + # print("bad file rename, fixing") + # os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) + # Already filtered and known good # else: # new_file_name = "_".join(file_parts[:-1]) + ".fits" # print("Removing filter check") # os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) continue - else: + elif file_name_end == "e": + if reprocess_errors: + new_file_name = "_".join(file_parts[:-1]) + ".fits" + os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) + files_to_process.append(os.path.abspath(os.path.join(root, new_file_name))) + else: + continue + elif file_name_end == "v1-0-2": files_to_process.append(os.path.abspath(os.path.join(root,f))) + else: + print(f"Error - Unexpeted FITS file name: {f}") for ftp in tqdm.tqdm(files_to_process, desc="Filtering files"): work_queue.put(ftp) except KeyboardInterrupt: