jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added error metadata to filtered filenames

Browse source
This commit is contained in:
Jeremy Karst 2024-05-24 21:23:37 -04:00
parent cebe35536e
commit 0241545d5e
1 changed files with 53 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
filter_FITS.py
View file

@ -2,6 +2,7 @@ import os
from collections import defaultdict
from multiprocessing import Queue, Process
import re
import traceback
import tqdm
import numpy as np
@ -40,7 +41,7 @@ def lowpriority():
measurement_names = ["094", "131", "171", "195", "284", "304"]
thresholds = [0.050, 0.10 , 1.00, 1.40, 0.90, 2.50]
max_center_skew = 5
max_center_skew = 7
circle_hough_thresh = 0.7
expected_dims = 1280
ratio_above_thresh_max = 0.5
@ -52,7 +53,7 @@ def filter_fits(work_queue):
job = work_queue.get()
if job is None:
return
measurement = job.split("_")[1].split("-")[-1][2:]
measurement = job.split("dr_suvi-l2-")[1].split("_")[0][2:]
idx = measurement_names.index(measurement)
data = fits.getdata(job)
assert data.shape[0] == expected_dims
@ -64,6 +65,15 @@ def filter_fits(work_queue):
if ratio_above_thresh > ratio_above_thresh_max:
print(f"Exceeded ratio_above_thresh_max, possible data corruption in file: {job}")
new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name)
# plt.figure(f"Data {measurement}")
# plt.imshow(data, cmap='jet')
# plt.figure(f"Circle")
# plt.imshow(filtered_data, cmap='jet')
# plt.show()
continue
hough_radii = np.arange(387, 394, 1)
@ -80,13 +90,17 @@ def filter_fits(work_queue):
if not found_circle:
print(f"Could not find valid solar disc in file: {job}")
# plt.figure(f"Data {measurement}")
# if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1)
# plt.imshow(data, cmap='jet')
# plt.figure(f"Circle")
# if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1)
# plt.imshow(filtered_data, cmap='jet')
# plt.show()
plt.figure(f"Data {measurement}")
if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1)
plt.imshow(data, cmap='jet')
plt.figure(f"Circle")
if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1)
plt.imshow(filtered_data, cmap='jet')
plt.show()
new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name)
continue
# We have validated this file, rename it appropriately
@ -98,20 +112,25 @@ def filter_fits(work_queue):
return
except Exception as e:
print(f"Error on file: {job} - {e}")
traceback.print_exception(e)
if __name__ == "__main__":
# stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
stored_fits_dirs = [r"Z:\NOAA GOES Data\Data\goes18\l2\data\suvi-l2-ci094\2024"]
stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
# stored_fits_dirs = [r"..\Data\goes16\l2\data"]
# stored_fits_dirs = [r"Z:\NOAA GOES Data\fits_test_2024"]
reprocess_errors = False
nworkers = 20
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"
nworkers = 20
work_queue = Queue(maxsize = nworkers)
workers = []
@ -134,22 +153,29 @@ if __name__ == "__main__":
if filename_tester.match(f):
file_parts = f.split("_")
file_name_end = file_parts[-1].split(".")[0]
if file_name_end == "e":
# Error file, already processed
continue
elif file_name_end == "f":
if file_parts[-2] == "f":
new_file_name = "_".join(file_parts[:-2]) + "_f.fits"
print("bad file rename, fixing")
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
if file_name_end == "f":
# if file_parts[-2] == "f":
# new_file_name = "_".join(file_parts[:-2]) + "_f.fits"
# print("bad file rename, fixing")
# os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
# Already filtered and known good
# else:
# new_file_name = "_".join(file_parts[:-1]) + ".fits"
# print("Removing filter check")
# os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
continue
elif file_name_end == "e":
if reprocess_errors:
new_file_name = "_".join(file_parts[:-1]) + ".fits"
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
files_to_process.append(os.path.abspath(os.path.join(root, new_file_name)))
else:
continue
elif file_name_end == "v1-0-2":
files_to_process.append(os.path.abspath(os.path.join(root,f)))
else:
print(f"Error - Unexpeted FITS file name: {f}")
for ftp in tqdm.tqdm(files_to_process, desc="Filtering files"):
work_queue.put(ftp)
except KeyboardInterrupt: