jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added error metadata to filtered filenames

Browse source
This commit is contained in:
Jeremy Karst 2024-05-24 21:23:37 -04:00
parent cebe35536e
commit 0241545d5e
1 changed files with 53 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
filter_FITS.py
View file

@ -2,6 +2,7 @@ import os
from collections import defaultdict from collections import defaultdict
from multiprocessing import Queue, Process from multiprocessing import Queue, Process
import re import re
import traceback
import tqdm import tqdm
import numpy as np import numpy as np
@ -40,7 +41,7 @@ def lowpriority():
measurement_names = ["094", "131", "171", "195", "284", "304"] measurement_names = ["094", "131", "171", "195", "284", "304"]
thresholds = [0.050, 0.10 , 1.00, 1.40, 0.90, 2.50] thresholds = [0.050, 0.10 , 1.00, 1.40, 0.90, 2.50]
max_center_skew = 5 max_center_skew = 7
circle_hough_thresh = 0.7 circle_hough_thresh = 0.7
expected_dims = 1280 expected_dims = 1280
ratio_above_thresh_max = 0.5 ratio_above_thresh_max = 0.5
@ -52,7 +53,7 @@ def filter_fits(work_queue):
job = work_queue.get() job = work_queue.get()
if job is None: if job is None:
return return
measurement = job.split("_")[1].split("-")[-1][2:] measurement = job.split("dr_suvi-l2-")[1].split("_")[0][2:]
idx = measurement_names.index(measurement) idx = measurement_names.index(measurement)
data = fits.getdata(job) data = fits.getdata(job)
assert data.shape[0] == expected_dims assert data.shape[0] == expected_dims
@ -64,6 +65,15 @@ def filter_fits(work_queue):
if ratio_above_thresh > ratio_above_thresh_max: if ratio_above_thresh > ratio_above_thresh_max:
print(f"Exceeded ratio_above_thresh_max, possible data corruption in file: {job}") print(f"Exceeded ratio_above_thresh_max, possible data corruption in file: {job}")
new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name)
# plt.figure(f"Data {measurement}")
# plt.imshow(data, cmap='jet')
# plt.figure(f"Circle")
# plt.imshow(filtered_data, cmap='jet')
# plt.show()
continue continue
hough_radii = np.arange(387, 394, 1) hough_radii = np.arange(387, 394, 1)
@ -80,13 +90,17 @@ def filter_fits(work_queue):
if not found_circle: if not found_circle:
print(f"Could not find valid solar disc in file: {job}") print(f"Could not find valid solar disc in file: {job}")
# plt.figure(f"Data {measurement}")
# if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1) plt.figure(f"Data {measurement}")
# plt.imshow(data, cmap='jet') if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1)
# plt.figure(f"Circle") plt.imshow(data, cmap='jet')
# if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1) plt.figure(f"Circle")
# plt.imshow(filtered_data, cmap='jet') if found_circle: cv.circle(filtered_data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), 0.5, 1)
# plt.show() plt.imshow(filtered_data, cmap='jet')
plt.show()
new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name)
continue continue
# We have validated this file, rename it appropriately # We have validated this file, rename it appropriately
@ -98,20 +112,25 @@ def filter_fits(work_queue):
return return
except Exception as e: except Exception as e:
print(f"Error on file: {job} - {e}") print(f"Error on file: {job} - {e}")
traceback.print_exception(e)
if __name__ == "__main__": if __name__ == "__main__":
# stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024", stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci131\2024", r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci171\2024", r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci195\2024", r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci284\2024", r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
# r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",] r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
stored_fits_dirs = [r"Z:\NOAA GOES Data\Data\goes18\l2\data\suvi-l2-ci094\2024"]
# stored_fits_dirs = [r"..\Data\goes16\l2\data"] # stored_fits_dirs = [r"..\Data\goes16\l2\data"]
# stored_fits_dirs = [r"Z:\NOAA GOES Data\fits_test_2024"]
reprocess_errors = False
nworkers = 20
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$" regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"
nworkers = 20
work_queue = Queue(maxsize = nworkers) work_queue = Queue(maxsize = nworkers)
workers = [] workers = []
@ -134,22 +153,29 @@ if __name__ == "__main__":
if filename_tester.match(f): if filename_tester.match(f):
file_parts = f.split("_") file_parts = f.split("_")
file_name_end = file_parts[-1].split(".")[0] file_name_end = file_parts[-1].split(".")[0]
if file_name_end == "e": if file_name_end == "f":
# Error file, already processed # if file_parts[-2] == "f":
continue # new_file_name = "_".join(file_parts[:-2]) + "_f.fits"
elif file_name_end == "f": # print("bad file rename, fixing")
if file_parts[-2] == "f": # os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
new_file_name = "_".join(file_parts[:-2]) + "_f.fits"
print("bad file rename, fixing")
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
# Already filtered and known good # Already filtered and known good
# else: # else:
# new_file_name = "_".join(file_parts[:-1]) + ".fits" # new_file_name = "_".join(file_parts[:-1]) + ".fits"
# print("Removing filter check") # print("Removing filter check")
# os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) # os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
continue continue
else: elif file_name_end == "e":
if reprocess_errors:
new_file_name = "_".join(file_parts[:-1]) + ".fits"
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
files_to_process.append(os.path.abspath(os.path.join(root, new_file_name)))
else:
continue
elif file_name_end == "v1-0-2":
files_to_process.append(os.path.abspath(os.path.join(root,f))) files_to_process.append(os.path.abspath(os.path.join(root,f)))
else:
print(f"Error - Unexpeted FITS file name: {f}")
for ftp in tqdm.tqdm(files_to_process, desc="Filtering files"): for ftp in tqdm.tqdm(files_to_process, desc="Filtering files"):
work_queue.put(ftp) work_queue.put(ftp)
except KeyboardInterrupt: except KeyboardInterrupt: