jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Minor cleanup to filter_FITS

Browse source
This commit is contained in:
Jeremy Karst 2024-05-29 13:47:46 -04:00
parent c365957204
commit a54d10efab
1 changed files with 24 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
filter_FITS.py
View file

@ -91,13 +91,13 @@ def filter_fits(work_queue):
if not found_circle: if not found_circle:
print(f"Could not find valid solar disc in file: {job}") print(f"Could not find valid solar disc in file: {job}")
plt.figure(f"Data {measurement}") # plt.figure(f"Data {measurement}")
if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1) # if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1)
plt.imshow(data, cmap='jet') # plt.imshow(data, cmap='jet')
plt.figure(f"Circle") # plt.figure(f"Circle")
if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1) # if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1)
plt.imshow(filtered_data, cmap='jet') # plt.imshow(filtered_data, cmap='jet')
plt.show() # plt.show()
new_name = job.split(".fits")[0] + "_e.fits" new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name) os.rename(job, new_name)
@ -116,17 +116,23 @@ def filter_fits(work_queue):
if __name__ == "__main__": if __name__ == "__main__":
stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024", stored_fits_dirs = [r"..\Data\goes16\l2\data\suvi-l2-ci094",
r"..\Data\goes18\l2\data\suvi-l2-ci131\2024", r"..\Data\goes16\l2\data\suvi-l2-ci131",
r"..\Data\goes18\l2\data\suvi-l2-ci171\2024", r"..\Data\goes16\l2\data\suvi-l2-ci171",
r"..\Data\goes18\l2\data\suvi-l2-ci195\2024", r"..\Data\goes16\l2\data\suvi-l2-ci195",
r"..\Data\goes18\l2\data\suvi-l2-ci284\2024", r"..\Data\goes16\l2\data\suvi-l2-ci284",
r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",] r"..\Data\goes16\l2\data\suvi-l2-ci304",
r"..\Data\goes18\l2\data\suvi-l2-ci094",
r"..\Data\goes18\l2\data\suvi-l2-ci131",
r"..\Data\goes18\l2\data\suvi-l2-ci171",
r"..\Data\goes18\l2\data\suvi-l2-ci195",
r"..\Data\goes18\l2\data\suvi-l2-ci284",
r"..\Data\goes18\l2\data\suvi-l2-ci304",]
# stored_fits_dirs = [r"..\Data\goes16\l2\data"] # stored_fits_dirs = [r"..\Data\goes16\l2\data"]
# stored_fits_dirs = [r"Z:\NOAA GOES Data\fits_test_2024"] # stored_fits_dirs = [r"Z:\NOAA GOES Data\fits_test_2024"]
reprocess_errors = False reprocess_errors = False
nworkers = 20 nworkers = 16
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$" regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"
@ -154,17 +160,7 @@ if __name__ == "__main__":
file_parts = f.split("_") file_parts = f.split("_")
file_name_end = file_parts[-1].split(".")[0] file_name_end = file_parts[-1].split(".")[0]
if file_name_end == "f": if file_name_end == "f":
# if file_parts[-2] == "f": continue # Already filtered from a previous run
# new_file_name = "_".join(file_parts[:-2]) + "_f.fits"
# print("bad file rename, fixing")
# os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
# Already filtered and known good
# else:
# new_file_name = "_".join(file_parts[:-1]) + ".fits"
# print("Removing filter check")
# os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
continue
elif file_name_end == "e": elif file_name_end == "e":
if reprocess_errors: if reprocess_errors:
new_file_name = "_".join(file_parts[:-1]) + ".fits" new_file_name = "_".join(file_parts[:-1]) + ".fits"
@ -172,10 +168,11 @@ if __name__ == "__main__":
files_to_process.append(os.path.abspath(os.path.join(root, new_file_name))) files_to_process.append(os.path.abspath(os.path.join(root, new_file_name)))
else: else:
continue continue
elif file_name_end == "v1-0-2": elif file_name_end == "v1-0-2": # This is the normal case for unprocessed data
files_to_process.append(os.path.abspath(os.path.join(root,f))) files_to_process.append(os.path.abspath(os.path.join(root,f)))
else: else:
print(f"Error - Unexpeted FITS file name: {f}") # print(f"Error - Unexpected FITS file name: {f}")
pass
for ftp in tqdm.tqdm(files_to_process, desc="Filtering files"): for ftp in tqdm.tqdm(files_to_process, desc="Filtering files"):
work_queue.put(ftp) work_queue.put(ftp)
except KeyboardInterrupt: except KeyboardInterrupt: