diff --git a/filter_FITS.py b/filter_FITS.py index 8de2ca5..6c0735d 100644 --- a/filter_FITS.py +++ b/filter_FITS.py @@ -91,13 +91,13 @@ def filter_fits(work_queue): if not found_circle: print(f"Could not find valid solar disc in file: {job}") - plt.figure(f"Data {measurement}") - if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1) - plt.imshow(data, cmap='jet') - plt.figure(f"Circle") - if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1) - plt.imshow(filtered_data, cmap='jet') - plt.show() + # plt.figure(f"Data {measurement}") + # if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1) + # plt.imshow(data, cmap='jet') + # plt.figure(f"Circle") + # if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1) + # plt.imshow(filtered_data, cmap='jet') + # plt.show() new_name = job.split(".fits")[0] + "_e.fits" os.rename(job, new_name) @@ -116,17 +116,23 @@ def filter_fits(work_queue): if __name__ == "__main__": - stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci131\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci171\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci195\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci284\2024", - r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",] + stored_fits_dirs = [r"..\Data\goes16\l2\data\suvi-l2-ci094", + r"..\Data\goes16\l2\data\suvi-l2-ci131", + r"..\Data\goes16\l2\data\suvi-l2-ci171", + r"..\Data\goes16\l2\data\suvi-l2-ci195", + r"..\Data\goes16\l2\data\suvi-l2-ci284", + r"..\Data\goes16\l2\data\suvi-l2-ci304", + r"..\Data\goes18\l2\data\suvi-l2-ci094", + r"..\Data\goes18\l2\data\suvi-l2-ci131", + r"..\Data\goes18\l2\data\suvi-l2-ci171", + r"..\Data\goes18\l2\data\suvi-l2-ci195", + r"..\Data\goes18\l2\data\suvi-l2-ci284", + r"..\Data\goes18\l2\data\suvi-l2-ci304",] # stored_fits_dirs = [r"..\Data\goes16\l2\data"] # stored_fits_dirs = [r"Z:\NOAA GOES Data\fits_test_2024"] reprocess_errors = False - nworkers = 20 + nworkers = 16 regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$" @@ -154,17 +160,7 @@ if __name__ == "__main__": file_parts = f.split("_") file_name_end = file_parts[-1].split(".")[0] if file_name_end == "f": - # if file_parts[-2] == "f": - # new_file_name = "_".join(file_parts[:-2]) + "_f.fits" - # print("bad file rename, fixing") - # os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) - - # Already filtered and known good - # else: - # new_file_name = "_".join(file_parts[:-1]) + ".fits" - # print("Removing filter check") - # os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) - continue + continue # Already filtered from a previous run elif file_name_end == "e": if reprocess_errors: new_file_name = "_".join(file_parts[:-1]) + ".fits" @@ -172,10 +168,11 @@ if __name__ == "__main__": files_to_process.append(os.path.abspath(os.path.join(root, new_file_name))) else: continue - elif file_name_end == "v1-0-2": + elif file_name_end == "v1-0-2": # This is the normal case for unprocessed data files_to_process.append(os.path.abspath(os.path.join(root,f))) else: - print(f"Error - Unexpeted FITS file name: {f}") + # print(f"Error - Unexpected FITS file name: {f}") + pass for ftp in tqdm.tqdm(files_to_process, desc="Filtering files"): work_queue.put(ftp) except KeyboardInterrupt: