jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixed a bug in file queing for filter

Browse source
This commit is contained in:
Jeremy Karst 2024-06-04 23:44:19 -04:00
parent d5545bc0e5
commit 5dcf714bf7
1 changed files with 22 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
filter_FITS.py
View file

@ -42,7 +42,7 @@ def lowpriority():
os.nice(1) os.nice(1)
measurement_names = ["094", "131", "171", "195", "284", "304"] measurement_names = ["094", "131", "171", "195", "284", "304"]
thresholds = [0.050, 0.10 , 1.00, 1.40, 0.90, 2.50] thresholds = [0.050, 0.10 , 1.00, 1.40, 0.93, 2.50]
max_center_skew = 7 max_center_skew = 7
circle_hough_thresh = 0.75 circle_hough_thresh = 0.75
expected_dims = 1280 expected_dims = 1280
@ -81,7 +81,7 @@ def filter_fits(work_queue):
hough_radii = np.arange(383, 393, 1) hough_radii = np.arange(383, 393, 1)
hough_res = hough_circle(filtered_data, hough_radii) hough_res = hough_circle(filtered_data, hough_radii)
vals, cxs, cys, rads = hough_circle_peaks(hough_res, hough_radii, threshold=circle_hough_thresh, total_num_peaks=10) vals, cxs, cys, rads = hough_circle_peaks(hough_res, hough_radii, threshold=circle_hough_thresh, total_num_peaks=20)
found_circle = None found_circle = None
for v, cx, cy, rad in zip(vals, cxs, cys, rads): for v, cx, cy, rad in zip(vals, cxs, cys, rads):
xskew = abs(639 - cx) xskew = abs(639 - cx)
@ -90,6 +90,7 @@ def filter_fits(work_queue):
found_circle = (cx, cy, rad) found_circle = (cx, cy, rad)
break break
# print(f"Plotting file: {job}")
# plt.figure(f"Data {measurement}") # plt.figure(f"Data {measurement}")
# if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1) # if found_circle: cv.circle(data, (int(found_circle[0]),int(found_circle[1])), int(found_circle[2]), float(np.max(np.max(data))), 1)
# plt.imshow(data, cmap='jet') # plt.imshow(data, cmap='jet')
@ -108,6 +109,7 @@ def filter_fits(work_queue):
# We have validated this file, rename it appropriately # We have validated this file, rename it appropriately
new_name = job.split(".fits")[0] + "_f.fits" new_name = job.split(".fits")[0] + "_f.fits"
os.rename(job, new_name) os.rename(job, new_name)
# print(f"Renamed: {job} -> {new_name}")
except KeyboardInterrupt: except KeyboardInterrupt:
@ -130,7 +132,8 @@ if __name__ == "__main__":
r"..\Data\goes18\l2\data\suvi-l2-ci195\2023", r"..\Data\goes18\l2\data\suvi-l2-ci195\2023",
r"..\Data\goes18\l2\data\suvi-l2-ci284\2023", r"..\Data\goes18\l2\data\suvi-l2-ci284\2023",
r"..\Data\goes18\l2\data\suvi-l2-ci304\2023",] r"..\Data\goes18\l2\data\suvi-l2-ci304\2023",]
# stored_fits_dirs = [r"..\Data\goes16\l2\data\suvi-l2-ci131\2023\06\24"]
# stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci284\2023\01\04"]
starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
@ -164,21 +167,23 @@ if __name__ == "__main__":
file_name_end = file_parts[-1].split(".")[0] file_name_end = file_parts[-1].split(".")[0]
measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp()) measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
if (measure_end_time >= starttime) and (measure_end_time < stoptime): if (measure_end_time >= starttime) and (measure_end_time < stoptime):
files_by_timestamp[measure_end_time].append(os.path.join(root,f)) if file_name_end == "f":
if file_name_end == "f": if file_parts[-2] == "f": # We have an accidentally double filtered file...
continue # Already filtered from a previous run new_file_name = "_".join(file_parts[:-1]) + ".fits"
elif file_name_end == "e": os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
if reprocess_errors: continue # Already filtered from a previous run
new_file_name = "_".join(file_parts[:-1]) + ".fits" elif file_name_end == "e":
os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) if reprocess_errors:
files_by_timestamp[measure_end_time].append(os.path.abspath(os.path.join(root, new_file_name))) new_file_name = "_".join(file_parts[:-1]) + ".fits"
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
files_by_timestamp[measure_end_time].append(os.path.join(root, new_file_name))
else:
continue
elif file_name_end.startswith("v1-0-"): # This is the normal case for unprocessed data
files_by_timestamp[measure_end_time].append(os.path.join(root,f))
else: else:
continue # print(f"Error - Unexpected FITS file name: {f}")
elif file_name_end == "v1-0-2": # This is the normal case for unprocessed data pass
files_by_timestamp[measure_end_time].append(os.path.abspath(os.path.join(root,f)))
else:
# print(f"Error - Unexpected FITS file name: {f}")
pass
sorted_times = sorted(list(files_by_timestamp.keys())) sorted_times = sorted(list(files_by_timestamp.keys()))
for st in tqdm.tqdm(sorted_times, desc="Filtering files"): for st in tqdm.tqdm(sorted_times, desc="Filtering files"):
for f in files_by_timestamp[st]: for f in files_by_timestamp[st]: