jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Updated fetcher and added .vscode folder to gitignore

Browse source
This commit is contained in:
Jeremy Karst 2026-08-28 01:39:18 -04:00
parent 756270cd1b
commit 830a7eb469
6 changed files with 1213 additions and 1178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
.vscode

140
ffmpeg_video.py
View file

@ -1,67 +1,73 @@
import subprocess import subprocess
import os import os
import shutil import shutil
import datetime import datetime
import time import time
import calendar import calendar
import numpy as np import numpy as np
from sortedcontainers import SortedDict from sortedcontainers import SortedDict
import tqdm import tqdm
from PIL import Image from PIL import Image
path_to_images = r"..\composite\goes18" years = [2024]
output_file = r"..\goes18_2023.mp4" sources = ["goes16", "goes18"]
interp_file = r"..\goes18_2023_interp.mp4" ffmpeg_path = r"..\ffmpeg.exe"
ffmpeg_path = r"..\ffmpeg.exe"
starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) for year in years:
stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) for source in sources:
min_file_size = 350000 # Detect and remove corrupted files by filtering by file size path_to_images = f"..\\composite\\{source}"
max_file_size = 450000 output_file = f"..\\{source}_{year}_nofilt.mp4"
encoding_crf = 16 interp_file = f"..\\{source}_{year}_interp_nofilt.mp4"
max_frame_interp = 120
starttime = calendar.timegm(datetime.datetime(year, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
# Set up ffmpeg to stream images from an input pipe stoptime = calendar.timegm(datetime.datetime(year+1, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
command_line = f'{ffmpeg_path} -y -f image2pipe -framerate 60 -i - -c:v libx264 -crf {encoding_crf} -preset veryfast {output_file}' min_file_size = 350000 # Detect and remove corrupted files by filtering by file size
print(command_line) max_file_size = 450000
p = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE) encoding_crf = 16
max_frame_interp = 120
black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
# Set up ffmpeg to stream images from an input pipe
files_by_time = SortedDict() command_line = f'{ffmpeg_path} -y -f image2pipe -framerate 60 -i - -c:v libx264 -crf {encoding_crf} -preset veryfast {output_file}'
for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and filtering files."): print(command_line)
for f in files: p = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
if f.endswith('.jpg'):
fpath = os.path.join(root, f) black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
fsize = os.path.getsize(fpath)
time_chunk = f.replace('-', '.').replace('_', '.').split('.')[1] files_by_time = SortedDict()
ftime = int(time_chunk) for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and filtering files."):
if (fsize > min_file_size) and (fsize < max_file_size) and (ftime > starttime) and (ftime < stoptime): for f in files:
files_by_time[ftime] = fpath if f.endswith('.jpg'):
fpath = os.path.join(root, f)
difftimes = np.diff(files_by_time.keys()) fsize = os.path.getsize(fpath)
unique, counts = np.unique(difftimes, return_counts=True) time_chunk = f.replace('-', '.').replace('_', '.').split('.')[1]
interval = unique[0] ftime = int(time_chunk)
assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval if (fsize > min_file_size) and (fsize < max_file_size) and (ftime > starttime) and (ftime < stoptime):
files_by_time[ftime] = fpath
prevtime = files_by_time.peekitem(0)[0] - interval
for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", total = len(files_by_time)): difftimes = np.diff(files_by_time.keys())
framejump = (t - prevtime) // interval unique, counts = np.unique(difftimes, return_counts=True)
if framejump < max_frame_interp: interval = unique[0]
for i in range(framejump - 1): # Fill black frames to fill gaps, these will later be interpolated with ffmpeg assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval
black_image.save(p.stdin, 'jpeg', quality = 95)
else: # Skip past intervals that are too large to fill reasonably prevtime = files_by_time.peekitem(0)[0] - interval
print(f"Detected a frame gap of: {framejump}!") for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", total = len(files_by_time)):
with open(f, 'rb') as fh: framejump = (t - prevtime) // interval
p.stdin.write(fh.read()) if framejump < max_frame_interp:
prevtime = t for i in range(framejump - 1): # Fill black frames to fill gaps, these will later be interpolated with ffmpeg
black_image.save(p.stdin, 'jpeg', quality = 95)
p.stdin.close() # Close the ffmpeg input pipe else: # Skip past intervals that are too large to fill reasonably
p.wait() # Wait for ffmpeg to finish encoding print(f"Detected a frame gap of: {framejump}!")
with open(f, 'rb') as fh:
command_line = f'{ffmpeg_path} -y -i {output_file} -vf blackframe=0,metadata=select:key=lavfi.blackframe.pblack:value=95:function=less,minterpolate=mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:me=fss -c:a copy -c:v libx264 -crf {encoding_crf} -preset veryfast {interp_file}' p.stdin.write(fh.read())
print(command_line) prevtime = t
pipe = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE).stdout p.stdin.close() # Close the ffmpeg input pipe
output = pipe.read().decode() p.wait() # Wait for ffmpeg to finish encoding
pipe.close()
command_line = f'{ffmpeg_path} -y -i {output_file} -vf blackframe=0,metadata=select:key=lavfi.blackframe.pblack:value=95:function=less,minterpolate=mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:me=fss -c:a copy -c:v libx264 -crf {encoding_crf} -preset veryfast {interp_file}'
print(command_line)
pipe = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE).stdout
output = pipe.read().decode()
pipe.close()

562
filter_FITS.py
View file

@ -1,281 +1,281 @@
import os import os
from collections import defaultdict from collections import defaultdict
from multiprocessing import Queue, Process from multiprocessing import Queue, Process
import re import re
import traceback import traceback
import datetime import datetime
import calendar import calendar
import tqdm import tqdm
import numpy as np import numpy as np
from matplotlib import pyplot as plt from matplotlib import pyplot as plt
from astropy.io import fits from astropy.io import fits
from skimage.transform import hough_circle, hough_circle_peaks from skimage.transform import hough_circle, hough_circle_peaks
from skimage.feature import canny from skimage.feature import canny
from skimage.filters import gaussian from skimage.filters import gaussian
from skimage.morphology import skeletonize from skimage.morphology import skeletonize
import cv2 as cv import cv2 as cv
def lowpriority(): def lowpriority():
""" Set the priority of the process to lowest possible.""" """ Set the priority of the process to lowest possible."""
import sys import sys
try: try:
sys.getwindowsversion() sys.getwindowsversion()
except AttributeError: except AttributeError:
isWindows = False isWindows = False
else: else:
isWindows = True isWindows = True
if isWindows: if isWindows:
import win32api,win32process,win32con # pywin32 import win32api,win32process,win32con # pywin32
pid = win32api.GetCurrentProcessId() pid = win32api.GetCurrentProcessId()
phandle = win32api.OpenProcess(win32con.PROCESS_ALL_ACCESS, True, pid) phandle = win32api.OpenProcess(win32con.PROCESS_ALL_ACCESS, True, pid)
win32process.SetPriorityClass(phandle, win32process.THREAD_PRIORITY_LOWEST) win32process.SetPriorityClass(phandle, win32process.THREAD_PRIORITY_LOWEST)
# win32process.SetPriorityClass(phandle, win32process.IDLE_PRIORITY_CLASS) # win32process.SetPriorityClass(phandle, win32process.IDLE_PRIORITY_CLASS)
# tid = win32api.GetCurrentThreadId() # tid = win32api.GetCurrentThreadId()
# thandle = win32api.OpenThread(win32con.PROCESS_ALL_ACCESS, True, tid) # thandle = win32api.OpenThread(win32con.PROCESS_ALL_ACCESS, True, tid)
# win32process.SetThreadPriority(thandle, win32process.THREAD_MODE_BACKGROUND_BEGIN) # win32process.SetThreadPriority(thandle, win32process.THREAD_MODE_BACKGROUND_BEGIN)
else: else:
import os import os
os.nice(19) os.nice(19)
measurement_names = ["094", "131", "171", "195", "284", "304"] measurement_names = ["094", "131", "171", "195", "284", "304"]
thresholds = [0.050, 0.10 , 1.00, 1.40, 1.0, 2.50] thresholds = [0.050, 0.10 , 1.00, 1.40, 1.0, 2.50]
max_center_skew = 7 max_center_skew = 7
expected_dims = 1280 expected_dims = 1280
half_dims = expected_dims // 2 half_dims = expected_dims // 2
valid_radii = [383, 394] valid_radii = [383, 394]
avg_radius = (valid_radii[0] + valid_radii[1]) // 2 avg_radius = (valid_radii[0] + valid_radii[1]) // 2
max_radius_error = 80 max_radius_error = 80
ratio_above_thresh_max = 0.4 ratio_above_thresh_max = 0.4
ratio_above_thresh_min = 0.1 ratio_above_thresh_min = 0.07
def filter_fits(work_queue): def filter_fits(work_queue):
lowpriority() lowpriority()
idealcircle = np.zeros((expected_dims // 2, expected_dims // 2)) idealcircle = np.zeros((expected_dims // 2, expected_dims // 2))
cv.circle(idealcircle, (expected_dims // 4, expected_dims // 4), avg_radius // 2, 1, -1) cv.circle(idealcircle, (expected_dims // 4, expected_dims // 4), avg_radius // 2, 1, -1)
idealcircle_axis = np.average(idealcircle, 0) idealcircle_axis = np.average(idealcircle, 0)
while True: while True:
try: try:
job = work_queue.get() job = work_queue.get()
if job is None: if job is None:
return return
measurement = job.split("dr_suvi-l2-")[1].split("_")[0][2:] measurement = job.split("dr_suvi-l2-")[1].split("_")[0][2:]
idx = measurement_names.index(measurement) idx = measurement_names.index(measurement)
try: try:
data = fits.getdata(job) data = fits.getdata(job)
except IndexError: # This can happen with a blank or corrupt HDU in the .fits file except IndexError: # This can happen with a blank or corrupt HDU in the .fits file
new_name = job.split(".fits")[0] + "_e.fits" new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name) os.rename(job, new_name)
continue continue
assert data.shape[0] == expected_dims assert data.shape[0] == expected_dims
assert data.shape[1] == expected_dims assert data.shape[1] == expected_dims
data = cv.resize(data, dsize=(expected_dims // 2, expected_dims // 2), interpolation=cv.INTER_LINEAR) data = cv.resize(data, dsize=(expected_dims // 2, expected_dims // 2), interpolation=cv.INTER_LINEAR)
filtered_data = np.copy(data) filtered_data = np.copy(data)
above_thresh_indexes = data > thresholds[idx] above_thresh_indexes = data > thresholds[idx]
filtered_data[above_thresh_indexes] = thresholds[idx] filtered_data[above_thresh_indexes] = thresholds[idx]
filtered_data /= thresholds[idx] filtered_data /= thresholds[idx]
ratio_above_thresh = np.count_nonzero(above_thresh_indexes) / data.shape[0] / data.shape[1] ratio_above_thresh = np.count_nonzero(above_thresh_indexes) / data.shape[0] / data.shape[1]
if ratio_above_thresh > ratio_above_thresh_max: if ratio_above_thresh > ratio_above_thresh_max:
print(f"Exceeded ratio_above_thresh_max, possible data corruption in file: {job}") print(f"Exceeded ratio_above_thresh_max, possible data corruption in file: {job}")
new_name = job.split(".fits")[0] + "_e.fits" new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name) os.rename(job, new_name)
plt.figure(figsize=[10.24, 7.68]) plt.figure(figsize=[10.24, 7.68])
plt.title(f"Exceeded ratio_above_thresh_max [{ratio_above_thresh:0.2f}]") plt.title(f"Exceeded ratio_above_thresh_max [{ratio_above_thresh:0.2f}]")
plt.imshow(filtered_data, cmap='jet') plt.imshow(filtered_data, cmap='jet')
plt.savefig(job.split(".fits")[0] + "_e.jpg") plt.savefig(job.split(".fits")[0] + "_e.jpg")
plt.close('all') plt.close('all')
continue continue
if ratio_above_thresh < ratio_above_thresh_min: if ratio_above_thresh < ratio_above_thresh_min:
print(f"Below ratio_above_thresh_min, possible bad data in file: {job}") print(f"Below ratio_above_thresh_min, possible bad data in file: {job}")
new_name = job.split(".fits")[0] + "_e.fits" new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name) os.rename(job, new_name)
plt.figure(figsize=[10.24, 7.68]) plt.figure(figsize=[10.24, 7.68])
plt.title(f"Below ratio_above_thresh_min [{ratio_above_thresh:0.2f}]") plt.title(f"Below ratio_above_thresh_min [{ratio_above_thresh:0.2f}]")
plt.imshow(filtered_data, cmap='jet') plt.imshow(filtered_data, cmap='jet')
plt.savefig(job.split(".fits")[0] + "_e.jpg") plt.savefig(job.split(".fits")[0] + "_e.jpg")
plt.close('all') plt.close('all')
continue continue
# Decide whether the data is valid based on solar disc placement and levels. # Decide whether the data is valid based on solar disc placement and levels.
xavg = np.average(filtered_data, 0) xavg = np.average(filtered_data, 0)
yavg = np.average(filtered_data, 1) yavg = np.average(filtered_data, 1)
axis_indices = list(range(half_dims)) axis_indices = list(range(half_dims))
# Center Estimation # Center Estimation
weighted_avg_center_x = np.average(axis_indices, 0, xavg) weighted_avg_center_x = np.average(axis_indices, 0, xavg)
weighted_avg_center_y = np.average(axis_indices, 0, yavg) weighted_avg_center_y = np.average(axis_indices, 0, yavg)
good_center = True good_center = True
if (half_dims // 2 - weighted_avg_center_x > max_center_skew) or (half_dims // 2 - weighted_avg_center_y > max_center_skew): if (half_dims // 2 - weighted_avg_center_x > max_center_skew) or (half_dims // 2 - weighted_avg_center_y > max_center_skew):
print(f"Could not find centered solar disc in file: {job}") print(f"Could not find centered solar disc in file: {job}")
good_center = False good_center = False
# Radius Estimation # Radius Estimation
edge_thresh = 0.98 edge_thresh = 0.98
high_edge_x = np.argmax(np.cumsum(xavg) > (np.sum(xavg) * edge_thresh)) high_edge_x = np.argmax(np.cumsum(xavg) > (np.sum(xavg) * edge_thresh))
low_edge_x = len(xavg) - np.argmax(np.cumsum(np.flip(xavg)) > (np.sum(xavg) * edge_thresh)) low_edge_x = len(xavg) - np.argmax(np.cumsum(np.flip(xavg)) > (np.sum(xavg) * edge_thresh))
high_edge_y = np.argmax(np.cumsum(yavg) > (np.sum(yavg) * edge_thresh)) high_edge_y = np.argmax(np.cumsum(yavg) > (np.sum(yavg) * edge_thresh))
low_edge_y = len(yavg) - np.argmax(np.cumsum(np.flip(yavg)) > (np.sum(yavg) * edge_thresh)) low_edge_y = len(yavg) - np.argmax(np.cumsum(np.flip(yavg)) > (np.sum(yavg) * edge_thresh))
good_radius = True good_radius = True
calc_radius_x = (high_edge_x - low_edge_x) # We do not divide by two because our image is downsized by half in each dimension calc_radius_x = (high_edge_x - low_edge_x) # We do not divide by two because our image is downsized by half in each dimension
calc_radius_y = (high_edge_y - low_edge_y) calc_radius_y = (high_edge_y - low_edge_y)
calc_radius = (calc_radius_x + calc_radius_y) / 2.0 calc_radius = (calc_radius_x + calc_radius_y) / 2.0
if abs(calc_radius - avg_radius) > max_radius_error: if abs(calc_radius - avg_radius) > max_radius_error:
print(f"Could not find correct solar radius in file: {job}") print(f"Could not find correct solar radius in file: {job}")
good_radius = False good_radius = False
# Goodness of fit estimation vs perfect disc # Goodness of fit estimation vs perfect disc
gof_x = np.sum(np.abs(xavg - idealcircle_axis)) / half_dims gof_x = np.sum(np.abs(xavg - idealcircle_axis)) / half_dims
gof_y = np.sum(np.abs(yavg - idealcircle_axis)) / half_dims gof_y = np.sum(np.abs(yavg - idealcircle_axis)) / half_dims
good_fit = True good_fit = True
if gof_x > 0.2 or gof_y > 0.2: if gof_x > 0.2 or gof_y > 0.2:
print(f"Could not find valid solar disc in file: {job}") print(f"Could not find valid solar disc in file: {job}")
good_fit = False good_fit = False
if good_center and good_radius and good_fit: if good_center and good_radius and good_fit:
# We have validated this file, rename it appropriately # We have validated this file, rename it appropriately
new_name = job.split(".fits")[0] + "_f.fits" new_name = job.split(".fits")[0] + "_f.fits"
os.rename(job, new_name) os.rename(job, new_name)
else: else:
new_name = job.split(".fits")[0] + "_e.fits" new_name = job.split(".fits")[0] + "_e.fits"
os.rename(job, new_name) os.rename(job, new_name)
# Plot Results # Plot Results
fig, axes = plt.subplots(2, 2, width_ratios=(0.2, 1), height_ratios=(0.2, 1), gridspec_kw={"hspace":0.0, "wspace": 0.0}, figsize = [8, 8]) fig, axes = plt.subplots(2, 2, width_ratios=(0.2, 1), height_ratios=(0.2, 1), gridspec_kw={"hspace":0.0, "wspace": 0.0}, figsize = [8, 8])
axes[0][0].axis('off') axes[0][0].axis('off')
axes[0][1].plot(axis_indices, xavg) axes[0][1].plot(axis_indices, xavg)
axes[0][1].plot(axis_indices, idealcircle_axis) axes[0][1].plot(axis_indices, idealcircle_axis)
# axes[0][1].fill_between(axis_indices, xavg, idealcircle_axis, hatch="//", edgecolor="red", facecolor="none") # axes[0][1].fill_between(axis_indices, xavg, idealcircle_axis, hatch="//", edgecolor="red", facecolor="none")
axes[1][0].plot(yavg, axis_indices) axes[1][0].plot(yavg, axis_indices)
axes[1][0].plot(idealcircle_axis, axis_indices) axes[1][0].plot(idealcircle_axis, axis_indices)
axes[1][0].invert_xaxis() axes[1][0].invert_xaxis()
axes[1][0].invert_yaxis() axes[1][0].invert_yaxis()
axes[1][1].imshow(filtered_data, aspect='auto', vmin=0, vmax=1) axes[1][1].imshow(filtered_data, aspect='auto', vmin=0, vmax=1)
axes[1][1].vlines([low_edge_x, high_edge_x], 0, half_dims) axes[1][1].vlines([low_edge_x, high_edge_x], 0, half_dims)
axes[1][1].hlines([low_edge_y, high_edge_y], 0, half_dims) axes[1][1].hlines([low_edge_y, high_edge_y], 0, half_dims)
axes[1][1].scatter(half_dims // 2, half_dims // 2, marker="o", linewidths = 1, alpha = 0.75, edgecolors="black", facecolors = "none") axes[1][1].scatter(half_dims // 2, half_dims // 2, marker="o", linewidths = 1, alpha = 0.75, edgecolors="black", facecolors = "none")
axes[1][1].scatter(weighted_avg_center_x, weighted_avg_center_y, marker = "x", alpha = 0.75, facecolors='red') axes[1][1].scatter(weighted_avg_center_x, weighted_avg_center_y, marker = "x", alpha = 0.75, facecolors='red')
# Set shared x axis between imshow and xdata plots # Set shared x axis between imshow and xdata plots
axes[0][1].xaxis.set_ticks_position("top") axes[0][1].xaxis.set_ticks_position("top")
axes[0][1].yaxis.set_ticks_position("right") axes[0][1].yaxis.set_ticks_position("right")
axes[0][1].sharex(axes[1][1]) axes[0][1].sharex(axes[1][1])
axes[1][0].sharey(axes[1][1]) axes[1][0].sharey(axes[1][1])
axes[1][1].yaxis.set_ticks_position("right") axes[1][1].yaxis.set_ticks_position("right")
plt.savefig(job.split(".fits")[0] + "_e.jpg") plt.savefig(job.split(".fits")[0] + "_e.jpg")
plt.close('all') plt.close('all')
except KeyboardInterrupt: except KeyboardInterrupt:
return return
except Exception as e: except Exception as e:
print(f"Error on file: {job} - {e}") print(f"Error on file: {job} - {e}")
traceback.print_exception(e) traceback.print_exception(e)
if __name__ == "__main__": if __name__ == "__main__":
stored_fits_dirs = [r"..\Data\goes16\l2\data\suvi-l2-ci094\2024", stored_fits_dirs = [r"..\Data\goes16\l2\data\suvi-l2-ci094\2024",
r"..\Data\goes16\l2\data\suvi-l2-ci131\2024", r"..\Data\goes16\l2\data\suvi-l2-ci131\2024",
r"..\Data\goes16\l2\data\suvi-l2-ci171\2024", r"..\Data\goes16\l2\data\suvi-l2-ci171\2024",
r"..\Data\goes16\l2\data\suvi-l2-ci195\2024", r"..\Data\goes16\l2\data\suvi-l2-ci195\2024",
r"..\Data\goes16\l2\data\suvi-l2-ci284\2024", r"..\Data\goes16\l2\data\suvi-l2-ci284\2024",
r"..\Data\goes16\l2\data\suvi-l2-ci304\2024", r"..\Data\goes16\l2\data\suvi-l2-ci304\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci094\2024", r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci131\2024", r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci171\2024", r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci195\2024", r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci284\2024", r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",] r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
# stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci284\2023\01\10"] # stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci284\2023\01\10"]
starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple()) stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
reprocess_errors = False reprocess_errors = True
nworkers = 16 nworkers = 16
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$" regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"
work_queue = Queue(maxsize = nworkers) work_queue = Queue(maxsize = nworkers)
workers = [] workers = []
for i in range(nworkers): for i in range(nworkers):
p = Process(target = filter_fits, args = (work_queue,), daemon=True) p = Process(target = filter_fits, args = (work_queue,), daemon=True)
p.start() p.start()
workers.append(p) workers.append(p)
lowpriority() lowpriority()
files_by_timestamp = defaultdict(list) files_by_timestamp = defaultdict(list)
try: try:
for stored_fits_dir in stored_fits_dirs: for stored_fits_dir in stored_fits_dirs:
filename_tester = re.compile(regex_filename) filename_tester = re.compile(regex_filename)
found_files = 0 found_files = 0
print(f"Searching for FITS files in: {stored_fits_dir}") print(f"Searching for FITS files in: {stored_fits_dir}")
for root, dirs, files in tqdm.tqdm(os.walk(stored_fits_dir), desc="Searching"): for root, dirs, files in tqdm.tqdm(os.walk(stored_fits_dir), desc="Searching"):
for f in files: for f in files:
if filename_tester.match(f): if filename_tester.match(f):
file_parts = f.split("_") file_parts = f.split("_")
file_name_end = file_parts[-1].split(".")[0] file_name_end = file_parts[-1].split(".")[0]
measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp()) measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
if (measure_end_time >= starttime) and (measure_end_time < stoptime): if (measure_end_time >= starttime) and (measure_end_time < stoptime):
if file_name_end == "f": if file_name_end == "f":
if file_parts[-2] == "f": # We have an accidentally double filtered file... if file_parts[-2] == "f": # We have an accidentally double filtered file...
new_file_name = "_".join(file_parts[:-1]) + ".fits" new_file_name = "_".join(file_parts[:-1]) + ".fits"
os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
continue # Already filtered from a previous run continue # Already filtered from a previous run
elif file_name_end == "e": elif file_name_end == "e":
if reprocess_errors: if reprocess_errors:
# Rename file to remove error designation # Rename file to remove error designation
new_file_name = "_".join(file_parts[:-1]) + ".fits" new_file_name = "_".join(file_parts[:-1]) + ".fits"
os.rename(os.path.join(root,f), os.path.join(root,new_file_name)) os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
# Check if error image exists and delete if needed # Check if error image exists and delete if needed
img_name = new_file_name.split(".fits")[0] + "_e.jpg" img_name = new_file_name.split(".fits")[0] + "_e.jpg"
try: try:
os.remove(os.path.join(root, img_name)) os.remove(os.path.join(root, img_name))
except: except:
pass pass
# Add job to queue # Add job to queue
files_by_timestamp[measure_end_time].append(os.path.join(root, new_file_name)) files_by_timestamp[measure_end_time].append(os.path.join(root, new_file_name))
else: else:
continue continue
elif file_name_end.startswith("v1-0-"): # This is the normal case for unprocessed data elif file_name_end.startswith("v1-0-"): # This is the normal case for unprocessed data
files_by_timestamp[measure_end_time].append(os.path.join(root,f)) files_by_timestamp[measure_end_time].append(os.path.join(root,f))
else: else:
# print(f"Error - Unexpected FITS file name: {f}") # print(f"Error - Unexpected FITS file name: {f}")
pass pass
sorted_times = sorted(list(files_by_timestamp.keys())) sorted_times = sorted(list(files_by_timestamp.keys()))
for st in tqdm.tqdm(sorted_times, desc="Filtering files"): for st in tqdm.tqdm(sorted_times, desc="Filtering files"):
for f in files_by_timestamp[st]: for f in files_by_timestamp[st]:
work_queue.put(f) work_queue.put(f)
except KeyboardInterrupt: except KeyboardInterrupt:
print("Finishing current jobs and exiting") print("Finishing current jobs and exiting")
for _ in range(nworkers): for _ in range(nworkers):
try: try:
work_queue.put(None, timeout=10.0) work_queue.put(None, timeout=10.0)
except: except:
break break
for w in workers: for w in workers:
w.join() w.join()

1151
merger_FITS.py
View file

File diff suppressed because it is too large Load diff

529
puller_fits.py
View file

@ -1,258 +1,273 @@
import os import os
import urllib.request import urllib.request
import urllib.parse import urllib.parse
import re import re
import time import time
import random import random
import datetime import datetime
import json import json
from threading import Thread from threading import Thread
import queue import queue
import math import math
from functools import partial from functools import partial
from queue import Empty from queue import Empty
import tqdm import tqdm
directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/" directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/"
stored_images_dir = r"..\Data" stored_images_dir = os.path.abspath(os.path.join("..", "Data"))
# directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/ephe-l2-orb1m/2017/12/" # directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/ephe-l2-orb1m/2017/12/"
# stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16/l2/data/ephe-l2-orb1m/2017/12/" # stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16/l2/data/ephe-l2-orb1m/2017/12/"
ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"] ignore_folder_names = ["Parent Directory", "l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
file_database_path = r"..\file_database.json" file_database_path = os.path.abspath(os.path.join("..", "file_database.json"))
fetch_interval = 0 # 60*60 fetch_interval = 0 # 60*60
nfetchworkers = 10 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time nfetchworkers = 2 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
ndownloadworkers = 4 # Be nice to the servers, this value is how many threads will be downloading files at the same time ndownloadworkers = 3 # Be nice to the servers, this value is how many threads will be downloading files at the same time
randomize_order = False randomize_order = False
links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace
times_regex_pattern = r'(?<=<td align="right">)(\d{4}-\d{2}-\d{2} \d{2}:\d{2})(?= )' # Find timestamps in UTC in the format YYYY-MM-DD HH:mm times_regex_pattern = r'(?<=<td align="right">)(\d{4}-\d{2}-\d{2} \d{2}:\d{2})(?= )' # Find timestamps in UTC in the format YYYY-MM-DD HH:mm
sizes_regex_pattern = r'(?<=\d{4}-\d{2}-\d{2} \d{2}:\d{2} <\/td><td align="right">)([ .\d]{1,3})([KMG]|- |0 | )(?=<\/td>)' # Find file size markers in one of many possible formats sizes_regex_pattern = r'(?<=\d{4}-\d{2}-\d{2} \d{2}:\d{2} <\/td><td align="right">)([ .\d]{1,3})([KMG]|- |0 | )(?=<\/td>)' # Find file size markers in one of many possible formats
links_matcher = re.compile(links_regex_pattern) links_matcher = re.compile(links_regex_pattern)
times_matcher = re.compile(times_regex_pattern) times_matcher = re.compile(times_regex_pattern)
sizes_matcher = re.compile(sizes_regex_pattern) sizes_matcher = re.compile(sizes_regex_pattern)
def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, randomize_order = True): def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, randomize_order = True):
while True: while True:
job = query_work_queue.get() job = query_work_queue.get()
if job == None: if job == None:
return return
url= job url= job
attempts = 0 attempts = 0
html_content = None html_content = None
while attempts < attempt_count: while attempts < attempt_count:
try: try:
with urllib.request.urlopen(url) as response: with urllib.request.urlopen(url) as response:
html_content = response.read().decode('utf-8') html_content = response.read().decode('utf-8')
break break
except Exception as e: except Exception as e:
print(f"Exception while fetching links: {e}") tqdm.tqdm.write(f"Exception while fetching links: {e}")
time.sleep(1 + random.random()) time.sleep(1 + random.random())
attempts += 1 attempts += 1
if (attempt_count > 1) and (attempts == attempt_count): if (attempt_count > 1) and (attempts == attempt_count):
print(f'\nAfter {attempt_count} retries, could not fetch: {url}') tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
return
if html_content is None: # We failed to fetch the link for some reason, continue to the next job
links = links_matcher.findall(html_content) continue
times = times_matcher.findall(html_content) else:
sizes = sizes_matcher.findall(html_content) links = links_matcher.findall(html_content)
for i in range(len(times)): times = times_matcher.findall(html_content)
dt = datetime.datetime.strptime(times[i].strip(), "%Y-%m-%d %H:%M") sizes = sizes_matcher.findall(html_content)
dt.replace(tzinfo=datetime.timezone.utc) for i in range(len(times)):
times[i] = time.mktime(dt.timetuple()) dt = datetime.datetime.strptime(times[i].strip(), "%Y-%m-%d %H:%M")
for i in range(len(sizes)): dt.replace(tzinfo=datetime.timezone.utc)
match sizes[i][1].strip(): times[i] = time.mktime(dt.timetuple())
case '-': for i in range(len(sizes)):
sizes[i] = 0 match sizes[i][1].strip():
case 'K': case '-':
sizes[i] = int(float(sizes[i][0].strip()) * 1024) sizes[i] = 0
case 'M': case 'K':
sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024) sizes[i] = int(float(sizes[i][0].strip()) * 1024)
case 'G': case 'M':
sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024 * 1024) sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024)
case "0": case 'G':
sizes[i] = 0 sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024 * 1024)
case "": case "0":
sizes[i] = int(float(sizes[i][0].strip())) sizes[i] = 0
case _: case "":
raise(ValueError(f"Unexpected symbol while parsing links page: {_}")) sizes[i] = int(float(sizes[i][0].strip()))
case _:
raise(ValueError(f"Unexpected symbol while parsing links page: {_}"))
if (len(links) != len(times)) or (len(times) != len(sizes)):
raise(ValueError("Links parsing error, mismatched numbers of links, times, or sizes!"))
if (len(links) != len(times)) or (len(times) != len(sizes)):
results = list(zip(links, times, sizes)) raise(ValueError("Links parsing error, mismatched numbers of links, times, or sizes!"))
if randomize_order:
random.shuffle(results) results = list(zip(links, times, sizes))
for _link, _time, _size in results: if randomize_order:
if _link.endswith("/"): random.shuffle(results)
if _link.split(r"/")[-2] in ignore_folder_names: for _link, _time, _size in results:
continue if _link.endswith("/"):
else: if _link.split(r"/")[-2] in ignore_folder_names:
query_work_queue.put(urllib.parse.urljoin(url,_link)) continue
else: else:
query_result_queue.put((urllib.parse.urljoin(url,_link), _time, _size)) # print(f"Queueing folder: {urllib.parse.urljoin(url,_link)}")
query_work_queue.put(urllib.parse.urljoin(url,_link))
else:
# Fetch file from url and store it to path, retrying on failure # print(f"Queueing file: {urllib.parse.urljoin(url,_link)}")
def file_download_worker(download_work_queue, download_result_queue, attempt_count = 2): query_result_queue.put((urllib.parse.urljoin(url,_link), _time, _size))
while True:
job = download_work_queue.get()
if job == None: # Fetch file from url and store it to path, retrying on failure
return def file_download_worker(download_work_queue, download_result_queue, attempt_count = 2):
url, path, t, s = job while True:
job = download_work_queue.get()
attempts = 0 if job == None:
while attempts < attempt_count: return
try: url, path, t, s = job
req = urllib.request.Request(url, data=None)
image_data = urllib.request.urlopen(req).read() # print(f"Next job: {url}, {path}, {t}, {s}")
if math.isclose(len(image_data), s, rel_tol=0.05): attempts = 0
os.makedirs(os.path.split(path)[0], exist_ok=True) while attempts < attempt_count:
open(path, 'wb').write(image_data) try:
download_result_queue.put((True, url, t)) starttime = time.time()
break req = urllib.request.Request(url, data=None)
else: image_data = urllib.request.urlopen(req, timeout=10.0).read()
raise ValueError("Downloaded file is the wrong size!") endtime = time.time()
except Exception as e: tqdm.tqdm.write(f"Downloaded {url} in {endtime-starttime:0.2f} s | {len(image_data)/1024/1024/(endtime-starttime):0.2f} MB/s")
if hasattr(e, "code") and e.code == 404: # This is expected if the file has been removed from the site (at least for swpc.noaa.gov) if math.isclose(len(image_data), s, rel_tol=0.05):
attempts += attempt_count os.makedirs(os.path.split(path)[0], exist_ok=True)
elif attempts == 0: open(path, 'wb').write(image_data)
print(f"\nA problem occurred on image: {url} | {e}") download_result_queue.put((True, url, t))
time.sleep(1 + random.random()) break
attempts += 1 else:
if (attempt_count > 1) and (attempts == attempt_count): raise ValueError("Downloaded file is the wrong size!")
print(f'\nAfter {attempt_count} retries, could not fetch: {url}') except Exception as e:
download_result_queue.put((False, url, t)) if hasattr(e, "code") and e.code == 404: # This is expected if the file has been removed from the site (at least for swpc.noaa.gov)
break attempts += attempt_count
elif (0 < attempts) and (attempts < attempt_count):
# tqdm.tqdm.write(f"\nA problem occurred on file: {url} | {e}")
if __name__ == "__main__": time.sleep(1 + random.random())
file_info_cache = {} attempts += 1
try: if (attempt_count > 1) and (attempts == attempt_count):
print(f"Attempting to load file records from cache: {file_database_path}") tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
with open(file_database_path, 'r') as f: tqdm.tqdm.write(f"Exception: {e}")
file_info_cache = json.loads(f.read()) download_result_queue.put((False, url, t))
print(f"File records loaded from cache: {len(file_info_cache)} records found.") break
except Exception as e:
print(f"Load failed, starting with empty cache")
file_info_cache = {} if __name__ == "__main__":
file_info_cache = {}
query_work_queue = queue.Queue() try:
query_result_queue = queue.Queue() print(f"Attempting to load file records from cache: {file_database_path}")
download_work_queue = queue.Queue() with open(file_database_path, 'r') as f:
download_result_queue = queue.Queue() file_info_cache = json.loads(f.read())
workers = [] print(f"File records loaded from cache: {len(file_info_cache)} records found.")
for _ in range(nfetchworkers): except Exception as e:
t = Thread(target=find_links_worker, args=(query_work_queue, query_result_queue, 3, randomize_order), daemon=True) print(f"Load failed, starting with empty cache")
t.start() file_info_cache = {}
workers.append(t)
for _ in range(ndownloadworkers): query_work_queue = queue.Queue()
t = Thread(target=file_download_worker, args=(download_work_queue, download_result_queue), daemon=True) query_result_queue = queue.Queue()
t.start() download_work_queue = queue.Queue(maxsize=ndownloadworkers)
workers.append(t) download_result_queue = queue.Queue(maxsize=ndownloadworkers)
workers = []
try: for _ in range(nfetchworkers):
while True: t = Thread(target=find_links_worker, args=(query_work_queue, query_result_queue, 3, randomize_order), daemon=True)
fetched_image_count = 0 t.start()
already_had_image_count = 0 workers.append(t)
failed_image_count = 0 for _ in range(ndownloadworkers):
urllen = len(directory_url) t = Thread(target=file_download_worker, args=(download_work_queue, download_result_queue), daemon=True)
t.start()
query_work_queue.put(directory_url) workers.append(t)
for l, t, s in tqdm.tqdm(iter(partial(query_result_queue.get, timeout=30.0), None), desc="Downloading files"):
# Collect completed jobs and record completion status try:
while True: while True:
try: fetched_image_count = 0
r_success, r_url, r_t = download_result_queue.get_nowait() already_had_image_count = 0
if r_success: failed_image_count = 0
fetched_image_count += 1 urllen = len(directory_url)
file_info_cache[r_url] = r_t
else: query_work_queue.put(directory_url)
failed_image_count += 1 for l, t, s in tqdm.tqdm(iter(partial(query_result_queue.get, timeout=30.0), None), desc="Downloading files"):
except queue.Empty: # Collect completed jobs and record completion status
break while True:
try:
file_portion_of_link = l[urllen:] r_success, r_url, r_t = download_result_queue.get_nowait()
filepath = os.path.join(stored_images_dir, file_portion_of_link.replace("/", os.sep)) if r_success:
# If we dont have the file or the file at the link is newer than the one we previously fetched fetched_image_count += 1
if (not (l in file_info_cache)) or t > file_info_cache[l]: file_info_cache[r_url] = r_t
if filepath.endswith(".fits"): else:
filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file failed_image_count += 1
filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file except queue.Empty:
if os.path.exists(filepath2) or os.path.exists(filepath3): # If the unfiltered filename exists, that case will be handled in the alternative code path in if os.path.exists(filepath): break
if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
try: file_portion_of_link = l[urllen:]
os.remove(filepath2) filepath = os.path.join(stored_images_dir, file_portion_of_link.replace("/", os.sep))
os.remove(filepath3) # If we dont have the file or the file at the link is newer than the one we previously fetched
except: if (not (l in file_info_cache)) or t > file_info_cache[l]:
pass if filepath.endswith(".fits"):
else: # If we have no record of this file, update the file info cache and don't redownload filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file
file_info_cache[l] = t filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file
already_had_image_count += 1 if os.path.exists(filepath2) or os.path.exists(filepath3): # If the unfiltered filename exists, that case will be handled in the alternative code path in if os.path.exists(filepath):
continue if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
if os.path.exists(filepath): try:
if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version. os.remove(filepath2)
if os.path.exists(filepath): os.remove(filepath3)
os.remove(filepath) except:
else: # If we have no record of this file, update the file info cache and don't redownload if fsize is right pass
fsize = os.path.getsize(filepath) else: # If we have no record of this file, update the file info cache and don't redownload
if math.isclose(fsize, s, rel_tol=0.05): file_info_cache[l] = t
file_info_cache[l] = t already_had_image_count += 1
already_had_image_count += 1 continue
continue if os.path.exists(filepath):
else: if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
print(f'Found a mismatched size on file: {filepath} Redownloading!') if os.path.exists(filepath):
download_work_queue.put((l, filepath, t, s)) os.remove(filepath)
else: else: # If we have no record of this file, update the file info cache and don't redownload if fsize is right
# We have a download record, confirm the file actually exists on disk fsize = os.path.getsize(filepath)
if os.path.exists(filepath): if math.isclose(fsize, s, rel_tol=0.05):
already_had_image_count += 1 file_info_cache[l] = t
elif filepath.endswith(".fits"): already_had_image_count += 1
filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file continue
filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file else:
if os.path.exists(filepath2) or os.path.exists(filepath3): tqdm.tqdm.write(f'Found a mismatched size on file: {filepath} Redownloading!')
already_had_image_count += 1 download_work_queue.put((l, filepath, t, s))
else: # We could not find the file on disk, queue for redownload else:
download_work_queue.put((l, filepath, t)) # We have a download record, confirm the file actually exists on disk
if os.path.exists(filepath):
with open(file_database_path, 'w') as f: already_had_image_count += 1
f.write(json.dumps(file_info_cache)) elif filepath.endswith(".fits"):
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}") filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file
if fetch_interval > 0: filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file
print(f"Run complete!, sleeping for {fetch_interval} seconds.") if os.path.exists(filepath2) or os.path.exists(filepath3):
with open(file_database_path, 'w') as f: already_had_image_count += 1
f.write(json.dumps(file_info_cache)) else: # We could not find the file on disk, queue for redownload
time.sleep(fetch_interval) download_work_queue.put((l, filepath, t))
else:
print("Run complete!, exiting...") with open(file_database_path, 'w') as f:
break f.write(json.dumps(file_info_cache))
except KeyboardInterrupt: print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
print("Saving file database and shutting down.") if fetch_interval > 0:
except Empty: print(f"Run complete!, sleeping for {fetch_interval} seconds.")
print("Work Complete, Shutting down...") with open(file_database_path, 'w') as f:
except Exception as e: f.write(json.dumps(file_info_cache))
print(f"Unhandled Exception during run: {e}") time.sleep(fetch_interval)
print("Shutting down") else:
print("Run complete!, exiting...")
with open(file_database_path, 'w') as f: break
f.write(json.dumps(file_info_cache)) except KeyboardInterrupt:
print("Saving file database and shutting down.")
for _ in range(nfetchworkers): except Empty:
try: print("Work Complete, Shutting down...")
query_work_queue.put(None, timeout=5.0) except Exception as e:
except: print(f"Unhandled Exception during run: {e}")
break print("Shutting down")
time.sleep(1) for _ in range(nfetchworkers):
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}") try:
query_work_queue.put(None)
for _ in range(ndownloadworkers): except:
try: break
download_work_queue.put(None, timeout=5.0)
except: time.sleep(1)
break print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
for w in workers: while len(download_work_queue.queue) > 0:
print(f"Waiting for {len(download_work_queue.queue)} downloads in queue...")
time.sleep(1)
for _ in range(ndownloadworkers):
try:
download_work_queue.put(None, timeout=5.0)
except:
break
with open(file_database_path, 'w') as f:
f.write(json.dumps(file_info_cache))
print("Waiting for workers to shutdown...")
for w in workers:
w.join(5.0) w.join(5.0)

8
requirements.txt Normal file
View file

@ -0,0 +1,8 @@
astropy
scikit-image
tqdm
numpy
opencv-python
palettable
matplotlib
pillow