Updated fetcher and added .vscode folder to gitignore
This commit is contained in:
parent
756270cd1b
commit
830a7eb469
6 changed files with 1213 additions and 1178 deletions
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
.vscode
|
||||
140
ffmpeg_video.py
140
ffmpeg_video.py
|
|
@ -1,67 +1,73 @@
|
|||
import subprocess
|
||||
import os
|
||||
import shutil
|
||||
import datetime
|
||||
import time
|
||||
import calendar
|
||||
|
||||
import numpy as np
|
||||
from sortedcontainers import SortedDict
|
||||
import tqdm
|
||||
from PIL import Image
|
||||
|
||||
path_to_images = r"..\composite\goes18"
|
||||
output_file = r"..\goes18_2023.mp4"
|
||||
interp_file = r"..\goes18_2023_interp.mp4"
|
||||
ffmpeg_path = r"..\ffmpeg.exe"
|
||||
starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
|
||||
stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
|
||||
min_file_size = 350000 # Detect and remove corrupted files by filtering by file size
|
||||
max_file_size = 450000
|
||||
encoding_crf = 16
|
||||
max_frame_interp = 120
|
||||
|
||||
# Set up ffmpeg to stream images from an input pipe
|
||||
command_line = f'{ffmpeg_path} -y -f image2pipe -framerate 60 -i - -c:v libx264 -crf {encoding_crf} -preset veryfast {output_file}'
|
||||
print(command_line)
|
||||
p = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
|
||||
|
||||
black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
|
||||
|
||||
files_by_time = SortedDict()
|
||||
for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and filtering files."):
|
||||
for f in files:
|
||||
if f.endswith('.jpg'):
|
||||
fpath = os.path.join(root, f)
|
||||
fsize = os.path.getsize(fpath)
|
||||
time_chunk = f.replace('-', '.').replace('_', '.').split('.')[1]
|
||||
ftime = int(time_chunk)
|
||||
if (fsize > min_file_size) and (fsize < max_file_size) and (ftime > starttime) and (ftime < stoptime):
|
||||
files_by_time[ftime] = fpath
|
||||
|
||||
difftimes = np.diff(files_by_time.keys())
|
||||
unique, counts = np.unique(difftimes, return_counts=True)
|
||||
interval = unique[0]
|
||||
assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval
|
||||
|
||||
prevtime = files_by_time.peekitem(0)[0] - interval
|
||||
for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", total = len(files_by_time)):
|
||||
framejump = (t - prevtime) // interval
|
||||
if framejump < max_frame_interp:
|
||||
for i in range(framejump - 1): # Fill black frames to fill gaps, these will later be interpolated with ffmpeg
|
||||
black_image.save(p.stdin, 'jpeg', quality = 95)
|
||||
else: # Skip past intervals that are too large to fill reasonably
|
||||
print(f"Detected a frame gap of: {framejump}!")
|
||||
with open(f, 'rb') as fh:
|
||||
p.stdin.write(fh.read())
|
||||
prevtime = t
|
||||
|
||||
p.stdin.close() # Close the ffmpeg input pipe
|
||||
p.wait() # Wait for ffmpeg to finish encoding
|
||||
|
||||
command_line = f'{ffmpeg_path} -y -i {output_file} -vf blackframe=0,metadata=select:key=lavfi.blackframe.pblack:value=95:function=less,minterpolate=mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:me=fss -c:a copy -c:v libx264 -crf {encoding_crf} -preset veryfast {interp_file}'
|
||||
print(command_line)
|
||||
|
||||
pipe = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE).stdout
|
||||
output = pipe.read().decode()
|
||||
pipe.close()
|
||||
import subprocess
|
||||
import os
|
||||
import shutil
|
||||
import datetime
|
||||
import time
|
||||
import calendar
|
||||
|
||||
import numpy as np
|
||||
from sortedcontainers import SortedDict
|
||||
import tqdm
|
||||
from PIL import Image
|
||||
|
||||
years = [2024]
|
||||
sources = ["goes16", "goes18"]
|
||||
ffmpeg_path = r"..\ffmpeg.exe"
|
||||
|
||||
for year in years:
|
||||
for source in sources:
|
||||
path_to_images = f"..\\composite\\{source}"
|
||||
output_file = f"..\\{source}_{year}_nofilt.mp4"
|
||||
interp_file = f"..\\{source}_{year}_interp_nofilt.mp4"
|
||||
|
||||
starttime = calendar.timegm(datetime.datetime(year, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
|
||||
stoptime = calendar.timegm(datetime.datetime(year+1, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
|
||||
min_file_size = 350000 # Detect and remove corrupted files by filtering by file size
|
||||
max_file_size = 450000
|
||||
encoding_crf = 16
|
||||
max_frame_interp = 120
|
||||
|
||||
# Set up ffmpeg to stream images from an input pipe
|
||||
command_line = f'{ffmpeg_path} -y -f image2pipe -framerate 60 -i - -c:v libx264 -crf {encoding_crf} -preset veryfast {output_file}'
|
||||
print(command_line)
|
||||
p = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
|
||||
|
||||
black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
|
||||
|
||||
files_by_time = SortedDict()
|
||||
for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and filtering files."):
|
||||
for f in files:
|
||||
if f.endswith('.jpg'):
|
||||
fpath = os.path.join(root, f)
|
||||
fsize = os.path.getsize(fpath)
|
||||
time_chunk = f.replace('-', '.').replace('_', '.').split('.')[1]
|
||||
ftime = int(time_chunk)
|
||||
if (fsize > min_file_size) and (fsize < max_file_size) and (ftime > starttime) and (ftime < stoptime):
|
||||
files_by_time[ftime] = fpath
|
||||
|
||||
difftimes = np.diff(files_by_time.keys())
|
||||
unique, counts = np.unique(difftimes, return_counts=True)
|
||||
interval = unique[0]
|
||||
assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval
|
||||
|
||||
prevtime = files_by_time.peekitem(0)[0] - interval
|
||||
for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", total = len(files_by_time)):
|
||||
framejump = (t - prevtime) // interval
|
||||
if framejump < max_frame_interp:
|
||||
for i in range(framejump - 1): # Fill black frames to fill gaps, these will later be interpolated with ffmpeg
|
||||
black_image.save(p.stdin, 'jpeg', quality = 95)
|
||||
else: # Skip past intervals that are too large to fill reasonably
|
||||
print(f"Detected a frame gap of: {framejump}!")
|
||||
with open(f, 'rb') as fh:
|
||||
p.stdin.write(fh.read())
|
||||
prevtime = t
|
||||
|
||||
p.stdin.close() # Close the ffmpeg input pipe
|
||||
p.wait() # Wait for ffmpeg to finish encoding
|
||||
|
||||
command_line = f'{ffmpeg_path} -y -i {output_file} -vf blackframe=0,metadata=select:key=lavfi.blackframe.pblack:value=95:function=less,minterpolate=mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:me=fss -c:a copy -c:v libx264 -crf {encoding_crf} -preset veryfast {interp_file}'
|
||||
print(command_line)
|
||||
|
||||
pipe = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE).stdout
|
||||
output = pipe.read().decode()
|
||||
pipe.close()
|
||||
562
filter_FITS.py
562
filter_FITS.py
|
|
@ -1,281 +1,281 @@
|
|||
import os
|
||||
from collections import defaultdict
|
||||
from multiprocessing import Queue, Process
|
||||
import re
|
||||
import traceback
|
||||
import datetime
|
||||
import calendar
|
||||
|
||||
import tqdm
|
||||
import numpy as np
|
||||
from matplotlib import pyplot as plt
|
||||
from astropy.io import fits
|
||||
from skimage.transform import hough_circle, hough_circle_peaks
|
||||
from skimage.feature import canny
|
||||
from skimage.filters import gaussian
|
||||
from skimage.morphology import skeletonize
|
||||
import cv2 as cv
|
||||
|
||||
def lowpriority():
|
||||
""" Set the priority of the process to lowest possible."""
|
||||
|
||||
import sys
|
||||
try:
|
||||
sys.getwindowsversion()
|
||||
except AttributeError:
|
||||
isWindows = False
|
||||
else:
|
||||
isWindows = True
|
||||
|
||||
if isWindows:
|
||||
import win32api,win32process,win32con # pywin32
|
||||
|
||||
pid = win32api.GetCurrentProcessId()
|
||||
phandle = win32api.OpenProcess(win32con.PROCESS_ALL_ACCESS, True, pid)
|
||||
win32process.SetPriorityClass(phandle, win32process.THREAD_PRIORITY_LOWEST)
|
||||
# win32process.SetPriorityClass(phandle, win32process.IDLE_PRIORITY_CLASS)
|
||||
# tid = win32api.GetCurrentThreadId()
|
||||
# thandle = win32api.OpenThread(win32con.PROCESS_ALL_ACCESS, True, tid)
|
||||
# win32process.SetThreadPriority(thandle, win32process.THREAD_MODE_BACKGROUND_BEGIN)
|
||||
else:
|
||||
import os
|
||||
|
||||
os.nice(19)
|
||||
|
||||
measurement_names = ["094", "131", "171", "195", "284", "304"]
|
||||
thresholds = [0.050, 0.10 , 1.00, 1.40, 1.0, 2.50]
|
||||
max_center_skew = 7
|
||||
expected_dims = 1280
|
||||
half_dims = expected_dims // 2
|
||||
valid_radii = [383, 394]
|
||||
avg_radius = (valid_radii[0] + valid_radii[1]) // 2
|
||||
max_radius_error = 80
|
||||
|
||||
ratio_above_thresh_max = 0.4
|
||||
ratio_above_thresh_min = 0.1
|
||||
|
||||
def filter_fits(work_queue):
|
||||
lowpriority()
|
||||
idealcircle = np.zeros((expected_dims // 2, expected_dims // 2))
|
||||
cv.circle(idealcircle, (expected_dims // 4, expected_dims // 4), avg_radius // 2, 1, -1)
|
||||
idealcircle_axis = np.average(idealcircle, 0)
|
||||
while True:
|
||||
try:
|
||||
job = work_queue.get()
|
||||
if job is None:
|
||||
return
|
||||
measurement = job.split("dr_suvi-l2-")[1].split("_")[0][2:]
|
||||
idx = measurement_names.index(measurement)
|
||||
try:
|
||||
data = fits.getdata(job)
|
||||
except IndexError: # This can happen with a blank or corrupt HDU in the .fits file
|
||||
new_name = job.split(".fits")[0] + "_e.fits"
|
||||
os.rename(job, new_name)
|
||||
continue
|
||||
assert data.shape[0] == expected_dims
|
||||
assert data.shape[1] == expected_dims
|
||||
data = cv.resize(data, dsize=(expected_dims // 2, expected_dims // 2), interpolation=cv.INTER_LINEAR)
|
||||
filtered_data = np.copy(data)
|
||||
above_thresh_indexes = data > thresholds[idx]
|
||||
filtered_data[above_thresh_indexes] = thresholds[idx]
|
||||
filtered_data /= thresholds[idx]
|
||||
ratio_above_thresh = np.count_nonzero(above_thresh_indexes) / data.shape[0] / data.shape[1]
|
||||
|
||||
if ratio_above_thresh > ratio_above_thresh_max:
|
||||
print(f"Exceeded ratio_above_thresh_max, possible data corruption in file: {job}")
|
||||
new_name = job.split(".fits")[0] + "_e.fits"
|
||||
os.rename(job, new_name)
|
||||
|
||||
plt.figure(figsize=[10.24, 7.68])
|
||||
plt.title(f"Exceeded ratio_above_thresh_max [{ratio_above_thresh:0.2f}]")
|
||||
plt.imshow(filtered_data, cmap='jet')
|
||||
plt.savefig(job.split(".fits")[0] + "_e.jpg")
|
||||
plt.close('all')
|
||||
|
||||
continue
|
||||
|
||||
if ratio_above_thresh < ratio_above_thresh_min:
|
||||
print(f"Below ratio_above_thresh_min, possible bad data in file: {job}")
|
||||
new_name = job.split(".fits")[0] + "_e.fits"
|
||||
os.rename(job, new_name)
|
||||
|
||||
plt.figure(figsize=[10.24, 7.68])
|
||||
plt.title(f"Below ratio_above_thresh_min [{ratio_above_thresh:0.2f}]")
|
||||
plt.imshow(filtered_data, cmap='jet')
|
||||
plt.savefig(job.split(".fits")[0] + "_e.jpg")
|
||||
plt.close('all')
|
||||
|
||||
continue
|
||||
|
||||
# Decide whether the data is valid based on solar disc placement and levels.
|
||||
xavg = np.average(filtered_data, 0)
|
||||
yavg = np.average(filtered_data, 1)
|
||||
axis_indices = list(range(half_dims))
|
||||
|
||||
# Center Estimation
|
||||
weighted_avg_center_x = np.average(axis_indices, 0, xavg)
|
||||
weighted_avg_center_y = np.average(axis_indices, 0, yavg)
|
||||
|
||||
good_center = True
|
||||
if (half_dims // 2 - weighted_avg_center_x > max_center_skew) or (half_dims // 2 - weighted_avg_center_y > max_center_skew):
|
||||
print(f"Could not find centered solar disc in file: {job}")
|
||||
good_center = False
|
||||
|
||||
# Radius Estimation
|
||||
edge_thresh = 0.98
|
||||
high_edge_x = np.argmax(np.cumsum(xavg) > (np.sum(xavg) * edge_thresh))
|
||||
low_edge_x = len(xavg) - np.argmax(np.cumsum(np.flip(xavg)) > (np.sum(xavg) * edge_thresh))
|
||||
high_edge_y = np.argmax(np.cumsum(yavg) > (np.sum(yavg) * edge_thresh))
|
||||
low_edge_y = len(yavg) - np.argmax(np.cumsum(np.flip(yavg)) > (np.sum(yavg) * edge_thresh))
|
||||
|
||||
good_radius = True
|
||||
calc_radius_x = (high_edge_x - low_edge_x) # We do not divide by two because our image is downsized by half in each dimension
|
||||
calc_radius_y = (high_edge_y - low_edge_y)
|
||||
calc_radius = (calc_radius_x + calc_radius_y) / 2.0
|
||||
if abs(calc_radius - avg_radius) > max_radius_error:
|
||||
print(f"Could not find correct solar radius in file: {job}")
|
||||
good_radius = False
|
||||
|
||||
# Goodness of fit estimation vs perfect disc
|
||||
gof_x = np.sum(np.abs(xavg - idealcircle_axis)) / half_dims
|
||||
gof_y = np.sum(np.abs(yavg - idealcircle_axis)) / half_dims
|
||||
|
||||
good_fit = True
|
||||
if gof_x > 0.2 or gof_y > 0.2:
|
||||
print(f"Could not find valid solar disc in file: {job}")
|
||||
good_fit = False
|
||||
|
||||
|
||||
if good_center and good_radius and good_fit:
|
||||
# We have validated this file, rename it appropriately
|
||||
new_name = job.split(".fits")[0] + "_f.fits"
|
||||
os.rename(job, new_name)
|
||||
else:
|
||||
new_name = job.split(".fits")[0] + "_e.fits"
|
||||
os.rename(job, new_name)
|
||||
|
||||
# Plot Results
|
||||
fig, axes = plt.subplots(2, 2, width_ratios=(0.2, 1), height_ratios=(0.2, 1), gridspec_kw={"hspace":0.0, "wspace": 0.0}, figsize = [8, 8])
|
||||
|
||||
axes[0][0].axis('off')
|
||||
|
||||
axes[0][1].plot(axis_indices, xavg)
|
||||
axes[0][1].plot(axis_indices, idealcircle_axis)
|
||||
# axes[0][1].fill_between(axis_indices, xavg, idealcircle_axis, hatch="//", edgecolor="red", facecolor="none")
|
||||
|
||||
axes[1][0].plot(yavg, axis_indices)
|
||||
axes[1][0].plot(idealcircle_axis, axis_indices)
|
||||
axes[1][0].invert_xaxis()
|
||||
axes[1][0].invert_yaxis()
|
||||
|
||||
axes[1][1].imshow(filtered_data, aspect='auto', vmin=0, vmax=1)
|
||||
axes[1][1].vlines([low_edge_x, high_edge_x], 0, half_dims)
|
||||
axes[1][1].hlines([low_edge_y, high_edge_y], 0, half_dims)
|
||||
axes[1][1].scatter(half_dims // 2, half_dims // 2, marker="o", linewidths = 1, alpha = 0.75, edgecolors="black", facecolors = "none")
|
||||
axes[1][1].scatter(weighted_avg_center_x, weighted_avg_center_y, marker = "x", alpha = 0.75, facecolors='red')
|
||||
|
||||
# Set shared x axis between imshow and xdata plots
|
||||
axes[0][1].xaxis.set_ticks_position("top")
|
||||
axes[0][1].yaxis.set_ticks_position("right")
|
||||
axes[0][1].sharex(axes[1][1])
|
||||
axes[1][0].sharey(axes[1][1])
|
||||
axes[1][1].yaxis.set_ticks_position("right")
|
||||
|
||||
plt.savefig(job.split(".fits")[0] + "_e.jpg")
|
||||
plt.close('all')
|
||||
|
||||
|
||||
except KeyboardInterrupt:
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"Error on file: {job} - {e}")
|
||||
traceback.print_exception(e)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stored_fits_dirs = [r"..\Data\goes16\l2\data\suvi-l2-ci094\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci131\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci171\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci195\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci284\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci304\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
|
||||
|
||||
# stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci284\2023\01\10"]
|
||||
|
||||
starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
|
||||
stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
|
||||
|
||||
reprocess_errors = False
|
||||
nworkers = 16
|
||||
|
||||
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"
|
||||
|
||||
work_queue = Queue(maxsize = nworkers)
|
||||
workers = []
|
||||
for i in range(nworkers):
|
||||
p = Process(target = filter_fits, args = (work_queue,), daemon=True)
|
||||
p.start()
|
||||
workers.append(p)
|
||||
|
||||
lowpriority()
|
||||
files_by_timestamp = defaultdict(list)
|
||||
try:
|
||||
for stored_fits_dir in stored_fits_dirs:
|
||||
filename_tester = re.compile(regex_filename)
|
||||
|
||||
found_files = 0
|
||||
print(f"Searching for FITS files in: {stored_fits_dir}")
|
||||
for root, dirs, files in tqdm.tqdm(os.walk(stored_fits_dir), desc="Searching"):
|
||||
for f in files:
|
||||
if filename_tester.match(f):
|
||||
file_parts = f.split("_")
|
||||
file_name_end = file_parts[-1].split(".")[0]
|
||||
measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
|
||||
if (measure_end_time >= starttime) and (measure_end_time < stoptime):
|
||||
if file_name_end == "f":
|
||||
if file_parts[-2] == "f": # We have an accidentally double filtered file...
|
||||
new_file_name = "_".join(file_parts[:-1]) + ".fits"
|
||||
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
|
||||
continue # Already filtered from a previous run
|
||||
elif file_name_end == "e":
|
||||
if reprocess_errors:
|
||||
# Rename file to remove error designation
|
||||
new_file_name = "_".join(file_parts[:-1]) + ".fits"
|
||||
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
|
||||
# Check if error image exists and delete if needed
|
||||
img_name = new_file_name.split(".fits")[0] + "_e.jpg"
|
||||
try:
|
||||
os.remove(os.path.join(root, img_name))
|
||||
except:
|
||||
pass
|
||||
# Add job to queue
|
||||
files_by_timestamp[measure_end_time].append(os.path.join(root, new_file_name))
|
||||
else:
|
||||
continue
|
||||
elif file_name_end.startswith("v1-0-"): # This is the normal case for unprocessed data
|
||||
files_by_timestamp[measure_end_time].append(os.path.join(root,f))
|
||||
else:
|
||||
# print(f"Error - Unexpected FITS file name: {f}")
|
||||
pass
|
||||
sorted_times = sorted(list(files_by_timestamp.keys()))
|
||||
for st in tqdm.tqdm(sorted_times, desc="Filtering files"):
|
||||
for f in files_by_timestamp[st]:
|
||||
work_queue.put(f)
|
||||
except KeyboardInterrupt:
|
||||
print("Finishing current jobs and exiting")
|
||||
|
||||
|
||||
for _ in range(nworkers):
|
||||
try:
|
||||
work_queue.put(None, timeout=10.0)
|
||||
except:
|
||||
break
|
||||
|
||||
for w in workers:
|
||||
w.join()
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from multiprocessing import Queue, Process
|
||||
import re
|
||||
import traceback
|
||||
import datetime
|
||||
import calendar
|
||||
|
||||
import tqdm
|
||||
import numpy as np
|
||||
from matplotlib import pyplot as plt
|
||||
from astropy.io import fits
|
||||
from skimage.transform import hough_circle, hough_circle_peaks
|
||||
from skimage.feature import canny
|
||||
from skimage.filters import gaussian
|
||||
from skimage.morphology import skeletonize
|
||||
import cv2 as cv
|
||||
|
||||
def lowpriority():
|
||||
""" Set the priority of the process to lowest possible."""
|
||||
|
||||
import sys
|
||||
try:
|
||||
sys.getwindowsversion()
|
||||
except AttributeError:
|
||||
isWindows = False
|
||||
else:
|
||||
isWindows = True
|
||||
|
||||
if isWindows:
|
||||
import win32api,win32process,win32con # pywin32
|
||||
|
||||
pid = win32api.GetCurrentProcessId()
|
||||
phandle = win32api.OpenProcess(win32con.PROCESS_ALL_ACCESS, True, pid)
|
||||
win32process.SetPriorityClass(phandle, win32process.THREAD_PRIORITY_LOWEST)
|
||||
# win32process.SetPriorityClass(phandle, win32process.IDLE_PRIORITY_CLASS)
|
||||
# tid = win32api.GetCurrentThreadId()
|
||||
# thandle = win32api.OpenThread(win32con.PROCESS_ALL_ACCESS, True, tid)
|
||||
# win32process.SetThreadPriority(thandle, win32process.THREAD_MODE_BACKGROUND_BEGIN)
|
||||
else:
|
||||
import os
|
||||
|
||||
os.nice(19)
|
||||
|
||||
measurement_names = ["094", "131", "171", "195", "284", "304"]
|
||||
thresholds = [0.050, 0.10 , 1.00, 1.40, 1.0, 2.50]
|
||||
max_center_skew = 7
|
||||
expected_dims = 1280
|
||||
half_dims = expected_dims // 2
|
||||
valid_radii = [383, 394]
|
||||
avg_radius = (valid_radii[0] + valid_radii[1]) // 2
|
||||
max_radius_error = 80
|
||||
|
||||
ratio_above_thresh_max = 0.4
|
||||
ratio_above_thresh_min = 0.07
|
||||
|
||||
def filter_fits(work_queue):
|
||||
lowpriority()
|
||||
idealcircle = np.zeros((expected_dims // 2, expected_dims // 2))
|
||||
cv.circle(idealcircle, (expected_dims // 4, expected_dims // 4), avg_radius // 2, 1, -1)
|
||||
idealcircle_axis = np.average(idealcircle, 0)
|
||||
while True:
|
||||
try:
|
||||
job = work_queue.get()
|
||||
if job is None:
|
||||
return
|
||||
measurement = job.split("dr_suvi-l2-")[1].split("_")[0][2:]
|
||||
idx = measurement_names.index(measurement)
|
||||
try:
|
||||
data = fits.getdata(job)
|
||||
except IndexError: # This can happen with a blank or corrupt HDU in the .fits file
|
||||
new_name = job.split(".fits")[0] + "_e.fits"
|
||||
os.rename(job, new_name)
|
||||
continue
|
||||
assert data.shape[0] == expected_dims
|
||||
assert data.shape[1] == expected_dims
|
||||
data = cv.resize(data, dsize=(expected_dims // 2, expected_dims // 2), interpolation=cv.INTER_LINEAR)
|
||||
filtered_data = np.copy(data)
|
||||
above_thresh_indexes = data > thresholds[idx]
|
||||
filtered_data[above_thresh_indexes] = thresholds[idx]
|
||||
filtered_data /= thresholds[idx]
|
||||
ratio_above_thresh = np.count_nonzero(above_thresh_indexes) / data.shape[0] / data.shape[1]
|
||||
|
||||
if ratio_above_thresh > ratio_above_thresh_max:
|
||||
print(f"Exceeded ratio_above_thresh_max, possible data corruption in file: {job}")
|
||||
new_name = job.split(".fits")[0] + "_e.fits"
|
||||
os.rename(job, new_name)
|
||||
|
||||
plt.figure(figsize=[10.24, 7.68])
|
||||
plt.title(f"Exceeded ratio_above_thresh_max [{ratio_above_thresh:0.2f}]")
|
||||
plt.imshow(filtered_data, cmap='jet')
|
||||
plt.savefig(job.split(".fits")[0] + "_e.jpg")
|
||||
plt.close('all')
|
||||
|
||||
continue
|
||||
|
||||
if ratio_above_thresh < ratio_above_thresh_min:
|
||||
print(f"Below ratio_above_thresh_min, possible bad data in file: {job}")
|
||||
new_name = job.split(".fits")[0] + "_e.fits"
|
||||
os.rename(job, new_name)
|
||||
|
||||
plt.figure(figsize=[10.24, 7.68])
|
||||
plt.title(f"Below ratio_above_thresh_min [{ratio_above_thresh:0.2f}]")
|
||||
plt.imshow(filtered_data, cmap='jet')
|
||||
plt.savefig(job.split(".fits")[0] + "_e.jpg")
|
||||
plt.close('all')
|
||||
|
||||
continue
|
||||
|
||||
# Decide whether the data is valid based on solar disc placement and levels.
|
||||
xavg = np.average(filtered_data, 0)
|
||||
yavg = np.average(filtered_data, 1)
|
||||
axis_indices = list(range(half_dims))
|
||||
|
||||
# Center Estimation
|
||||
weighted_avg_center_x = np.average(axis_indices, 0, xavg)
|
||||
weighted_avg_center_y = np.average(axis_indices, 0, yavg)
|
||||
|
||||
good_center = True
|
||||
if (half_dims // 2 - weighted_avg_center_x > max_center_skew) or (half_dims // 2 - weighted_avg_center_y > max_center_skew):
|
||||
print(f"Could not find centered solar disc in file: {job}")
|
||||
good_center = False
|
||||
|
||||
# Radius Estimation
|
||||
edge_thresh = 0.98
|
||||
high_edge_x = np.argmax(np.cumsum(xavg) > (np.sum(xavg) * edge_thresh))
|
||||
low_edge_x = len(xavg) - np.argmax(np.cumsum(np.flip(xavg)) > (np.sum(xavg) * edge_thresh))
|
||||
high_edge_y = np.argmax(np.cumsum(yavg) > (np.sum(yavg) * edge_thresh))
|
||||
low_edge_y = len(yavg) - np.argmax(np.cumsum(np.flip(yavg)) > (np.sum(yavg) * edge_thresh))
|
||||
|
||||
good_radius = True
|
||||
calc_radius_x = (high_edge_x - low_edge_x) # We do not divide by two because our image is downsized by half in each dimension
|
||||
calc_radius_y = (high_edge_y - low_edge_y)
|
||||
calc_radius = (calc_radius_x + calc_radius_y) / 2.0
|
||||
if abs(calc_radius - avg_radius) > max_radius_error:
|
||||
print(f"Could not find correct solar radius in file: {job}")
|
||||
good_radius = False
|
||||
|
||||
# Goodness of fit estimation vs perfect disc
|
||||
gof_x = np.sum(np.abs(xavg - idealcircle_axis)) / half_dims
|
||||
gof_y = np.sum(np.abs(yavg - idealcircle_axis)) / half_dims
|
||||
|
||||
good_fit = True
|
||||
if gof_x > 0.2 or gof_y > 0.2:
|
||||
print(f"Could not find valid solar disc in file: {job}")
|
||||
good_fit = False
|
||||
|
||||
|
||||
if good_center and good_radius and good_fit:
|
||||
# We have validated this file, rename it appropriately
|
||||
new_name = job.split(".fits")[0] + "_f.fits"
|
||||
os.rename(job, new_name)
|
||||
else:
|
||||
new_name = job.split(".fits")[0] + "_e.fits"
|
||||
os.rename(job, new_name)
|
||||
|
||||
# Plot Results
|
||||
fig, axes = plt.subplots(2, 2, width_ratios=(0.2, 1), height_ratios=(0.2, 1), gridspec_kw={"hspace":0.0, "wspace": 0.0}, figsize = [8, 8])
|
||||
|
||||
axes[0][0].axis('off')
|
||||
|
||||
axes[0][1].plot(axis_indices, xavg)
|
||||
axes[0][1].plot(axis_indices, idealcircle_axis)
|
||||
# axes[0][1].fill_between(axis_indices, xavg, idealcircle_axis, hatch="//", edgecolor="red", facecolor="none")
|
||||
|
||||
axes[1][0].plot(yavg, axis_indices)
|
||||
axes[1][0].plot(idealcircle_axis, axis_indices)
|
||||
axes[1][0].invert_xaxis()
|
||||
axes[1][0].invert_yaxis()
|
||||
|
||||
axes[1][1].imshow(filtered_data, aspect='auto', vmin=0, vmax=1)
|
||||
axes[1][1].vlines([low_edge_x, high_edge_x], 0, half_dims)
|
||||
axes[1][1].hlines([low_edge_y, high_edge_y], 0, half_dims)
|
||||
axes[1][1].scatter(half_dims // 2, half_dims // 2, marker="o", linewidths = 1, alpha = 0.75, edgecolors="black", facecolors = "none")
|
||||
axes[1][1].scatter(weighted_avg_center_x, weighted_avg_center_y, marker = "x", alpha = 0.75, facecolors='red')
|
||||
|
||||
# Set shared x axis between imshow and xdata plots
|
||||
axes[0][1].xaxis.set_ticks_position("top")
|
||||
axes[0][1].yaxis.set_ticks_position("right")
|
||||
axes[0][1].sharex(axes[1][1])
|
||||
axes[1][0].sharey(axes[1][1])
|
||||
axes[1][1].yaxis.set_ticks_position("right")
|
||||
|
||||
plt.savefig(job.split(".fits")[0] + "_e.jpg")
|
||||
plt.close('all')
|
||||
|
||||
|
||||
except KeyboardInterrupt:
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"Error on file: {job} - {e}")
|
||||
traceback.print_exception(e)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stored_fits_dirs = [r"..\Data\goes16\l2\data\suvi-l2-ci094\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci131\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci171\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci195\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci284\2024",
|
||||
r"..\Data\goes16\l2\data\suvi-l2-ci304\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci094\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci131\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci171\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci195\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci284\2024",
|
||||
r"..\Data\goes18\l2\data\suvi-l2-ci304\2024",]
|
||||
|
||||
# stored_fits_dirs = [r"..\Data\goes18\l2\data\suvi-l2-ci284\2023\01\10"]
|
||||
|
||||
starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
|
||||
stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
|
||||
|
||||
reprocess_errors = True
|
||||
nworkers = 16
|
||||
|
||||
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"
|
||||
|
||||
work_queue = Queue(maxsize = nworkers)
|
||||
workers = []
|
||||
for i in range(nworkers):
|
||||
p = Process(target = filter_fits, args = (work_queue,), daemon=True)
|
||||
p.start()
|
||||
workers.append(p)
|
||||
|
||||
lowpriority()
|
||||
files_by_timestamp = defaultdict(list)
|
||||
try:
|
||||
for stored_fits_dir in stored_fits_dirs:
|
||||
filename_tester = re.compile(regex_filename)
|
||||
|
||||
found_files = 0
|
||||
print(f"Searching for FITS files in: {stored_fits_dir}")
|
||||
for root, dirs, files in tqdm.tqdm(os.walk(stored_fits_dir), desc="Searching"):
|
||||
for f in files:
|
||||
if filename_tester.match(f):
|
||||
file_parts = f.split("_")
|
||||
file_name_end = file_parts[-1].split(".")[0]
|
||||
measure_end_time = int(datetime.datetime.strptime(file_parts[4][1:16] + " +0000", "%Y%m%dT%H%M%S %z").timestamp())
|
||||
if (measure_end_time >= starttime) and (measure_end_time < stoptime):
|
||||
if file_name_end == "f":
|
||||
if file_parts[-2] == "f": # We have an accidentally double filtered file...
|
||||
new_file_name = "_".join(file_parts[:-1]) + ".fits"
|
||||
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
|
||||
continue # Already filtered from a previous run
|
||||
elif file_name_end == "e":
|
||||
if reprocess_errors:
|
||||
# Rename file to remove error designation
|
||||
new_file_name = "_".join(file_parts[:-1]) + ".fits"
|
||||
os.rename(os.path.join(root,f), os.path.join(root,new_file_name))
|
||||
# Check if error image exists and delete if needed
|
||||
img_name = new_file_name.split(".fits")[0] + "_e.jpg"
|
||||
try:
|
||||
os.remove(os.path.join(root, img_name))
|
||||
except:
|
||||
pass
|
||||
# Add job to queue
|
||||
files_by_timestamp[measure_end_time].append(os.path.join(root, new_file_name))
|
||||
else:
|
||||
continue
|
||||
elif file_name_end.startswith("v1-0-"): # This is the normal case for unprocessed data
|
||||
files_by_timestamp[measure_end_time].append(os.path.join(root,f))
|
||||
else:
|
||||
# print(f"Error - Unexpected FITS file name: {f}")
|
||||
pass
|
||||
sorted_times = sorted(list(files_by_timestamp.keys()))
|
||||
for st in tqdm.tqdm(sorted_times, desc="Filtering files"):
|
||||
for f in files_by_timestamp[st]:
|
||||
work_queue.put(f)
|
||||
except KeyboardInterrupt:
|
||||
print("Finishing current jobs and exiting")
|
||||
|
||||
|
||||
for _ in range(nworkers):
|
||||
try:
|
||||
work_queue.put(None, timeout=10.0)
|
||||
except:
|
||||
break
|
||||
|
||||
for w in workers:
|
||||
w.join()
|
||||
|
|
|
|||
1151
merger_FITS.py
1151
merger_FITS.py
File diff suppressed because it is too large
Load diff
529
puller_fits.py
529
puller_fits.py
|
|
@ -1,258 +1,273 @@
|
|||
import os
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import re
|
||||
import time
|
||||
import random
|
||||
import datetime
|
||||
import json
|
||||
from threading import Thread
|
||||
import queue
|
||||
import math
|
||||
from functools import partial
|
||||
from queue import Empty
|
||||
|
||||
import tqdm
|
||||
|
||||
directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/"
|
||||
stored_images_dir = r"..\Data"
|
||||
# directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/ephe-l2-orb1m/2017/12/"
|
||||
# stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16/l2/data/ephe-l2-orb1m/2017/12/"
|
||||
ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
|
||||
file_database_path = r"..\file_database.json"
|
||||
fetch_interval = 0 # 60*60
|
||||
nfetchworkers = 10 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
|
||||
ndownloadworkers = 4 # Be nice to the servers, this value is how many threads will be downloading files at the same time
|
||||
randomize_order = False
|
||||
|
||||
links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace
|
||||
times_regex_pattern = r'(?<=<td align="right">)(\d{4}-\d{2}-\d{2} \d{2}:\d{2})(?= )' # Find timestamps in UTC in the format YYYY-MM-DD HH:mm
|
||||
sizes_regex_pattern = r'(?<=\d{4}-\d{2}-\d{2} \d{2}:\d{2} <\/td><td align="right">)([ .\d]{1,3})([KMG]|- |0 | )(?=<\/td>)' # Find file size markers in one of many possible formats
|
||||
links_matcher = re.compile(links_regex_pattern)
|
||||
times_matcher = re.compile(times_regex_pattern)
|
||||
sizes_matcher = re.compile(sizes_regex_pattern)
|
||||
|
||||
def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, randomize_order = True):
|
||||
while True:
|
||||
job = query_work_queue.get()
|
||||
if job == None:
|
||||
return
|
||||
url= job
|
||||
|
||||
attempts = 0
|
||||
html_content = None
|
||||
while attempts < attempt_count:
|
||||
try:
|
||||
with urllib.request.urlopen(url) as response:
|
||||
html_content = response.read().decode('utf-8')
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"Exception while fetching links: {e}")
|
||||
time.sleep(1 + random.random())
|
||||
attempts += 1
|
||||
if (attempt_count > 1) and (attempts == attempt_count):
|
||||
print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
|
||||
return
|
||||
|
||||
links = links_matcher.findall(html_content)
|
||||
times = times_matcher.findall(html_content)
|
||||
sizes = sizes_matcher.findall(html_content)
|
||||
for i in range(len(times)):
|
||||
dt = datetime.datetime.strptime(times[i].strip(), "%Y-%m-%d %H:%M")
|
||||
dt.replace(tzinfo=datetime.timezone.utc)
|
||||
times[i] = time.mktime(dt.timetuple())
|
||||
for i in range(len(sizes)):
|
||||
match sizes[i][1].strip():
|
||||
case '-':
|
||||
sizes[i] = 0
|
||||
case 'K':
|
||||
sizes[i] = int(float(sizes[i][0].strip()) * 1024)
|
||||
case 'M':
|
||||
sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024)
|
||||
case 'G':
|
||||
sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024 * 1024)
|
||||
case "0":
|
||||
sizes[i] = 0
|
||||
case "":
|
||||
sizes[i] = int(float(sizes[i][0].strip()))
|
||||
case _:
|
||||
raise(ValueError(f"Unexpected symbol while parsing links page: {_}"))
|
||||
|
||||
|
||||
if (len(links) != len(times)) or (len(times) != len(sizes)):
|
||||
raise(ValueError("Links parsing error, mismatched numbers of links, times, or sizes!"))
|
||||
|
||||
results = list(zip(links, times, sizes))
|
||||
if randomize_order:
|
||||
random.shuffle(results)
|
||||
for _link, _time, _size in results:
|
||||
if _link.endswith("/"):
|
||||
if _link.split(r"/")[-2] in ignore_folder_names:
|
||||
continue
|
||||
else:
|
||||
query_work_queue.put(urllib.parse.urljoin(url,_link))
|
||||
else:
|
||||
query_result_queue.put((urllib.parse.urljoin(url,_link), _time, _size))
|
||||
|
||||
|
||||
# Fetch file from url and store it to path, retrying on failure
|
||||
def file_download_worker(download_work_queue, download_result_queue, attempt_count = 2):
|
||||
while True:
|
||||
job = download_work_queue.get()
|
||||
if job == None:
|
||||
return
|
||||
url, path, t, s = job
|
||||
|
||||
attempts = 0
|
||||
while attempts < attempt_count:
|
||||
try:
|
||||
req = urllib.request.Request(url, data=None)
|
||||
image_data = urllib.request.urlopen(req).read()
|
||||
if math.isclose(len(image_data), s, rel_tol=0.05):
|
||||
os.makedirs(os.path.split(path)[0], exist_ok=True)
|
||||
open(path, 'wb').write(image_data)
|
||||
download_result_queue.put((True, url, t))
|
||||
break
|
||||
else:
|
||||
raise ValueError("Downloaded file is the wrong size!")
|
||||
except Exception as e:
|
||||
if hasattr(e, "code") and e.code == 404: # This is expected if the file has been removed from the site (at least for swpc.noaa.gov)
|
||||
attempts += attempt_count
|
||||
elif attempts == 0:
|
||||
print(f"\nA problem occurred on image: {url} | {e}")
|
||||
time.sleep(1 + random.random())
|
||||
attempts += 1
|
||||
if (attempt_count > 1) and (attempts == attempt_count):
|
||||
print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
|
||||
download_result_queue.put((False, url, t))
|
||||
break
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_info_cache = {}
|
||||
try:
|
||||
print(f"Attempting to load file records from cache: {file_database_path}")
|
||||
with open(file_database_path, 'r') as f:
|
||||
file_info_cache = json.loads(f.read())
|
||||
print(f"File records loaded from cache: {len(file_info_cache)} records found.")
|
||||
except Exception as e:
|
||||
print(f"Load failed, starting with empty cache")
|
||||
file_info_cache = {}
|
||||
|
||||
query_work_queue = queue.Queue()
|
||||
query_result_queue = queue.Queue()
|
||||
download_work_queue = queue.Queue()
|
||||
download_result_queue = queue.Queue()
|
||||
workers = []
|
||||
for _ in range(nfetchworkers):
|
||||
t = Thread(target=find_links_worker, args=(query_work_queue, query_result_queue, 3, randomize_order), daemon=True)
|
||||
t.start()
|
||||
workers.append(t)
|
||||
for _ in range(ndownloadworkers):
|
||||
t = Thread(target=file_download_worker, args=(download_work_queue, download_result_queue), daemon=True)
|
||||
t.start()
|
||||
workers.append(t)
|
||||
|
||||
try:
|
||||
while True:
|
||||
fetched_image_count = 0
|
||||
already_had_image_count = 0
|
||||
failed_image_count = 0
|
||||
urllen = len(directory_url)
|
||||
|
||||
query_work_queue.put(directory_url)
|
||||
for l, t, s in tqdm.tqdm(iter(partial(query_result_queue.get, timeout=30.0), None), desc="Downloading files"):
|
||||
# Collect completed jobs and record completion status
|
||||
while True:
|
||||
try:
|
||||
r_success, r_url, r_t = download_result_queue.get_nowait()
|
||||
if r_success:
|
||||
fetched_image_count += 1
|
||||
file_info_cache[r_url] = r_t
|
||||
else:
|
||||
failed_image_count += 1
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
file_portion_of_link = l[urllen:]
|
||||
filepath = os.path.join(stored_images_dir, file_portion_of_link.replace("/", os.sep))
|
||||
# If we dont have the file or the file at the link is newer than the one we previously fetched
|
||||
if (not (l in file_info_cache)) or t > file_info_cache[l]:
|
||||
if filepath.endswith(".fits"):
|
||||
filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file
|
||||
filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file
|
||||
if os.path.exists(filepath2) or os.path.exists(filepath3): # If the unfiltered filename exists, that case will be handled in the alternative code path in if os.path.exists(filepath):
|
||||
if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
|
||||
try:
|
||||
os.remove(filepath2)
|
||||
os.remove(filepath3)
|
||||
except:
|
||||
pass
|
||||
else: # If we have no record of this file, update the file info cache and don't redownload
|
||||
file_info_cache[l] = t
|
||||
already_had_image_count += 1
|
||||
continue
|
||||
if os.path.exists(filepath):
|
||||
if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
|
||||
if os.path.exists(filepath):
|
||||
os.remove(filepath)
|
||||
else: # If we have no record of this file, update the file info cache and don't redownload if fsize is right
|
||||
fsize = os.path.getsize(filepath)
|
||||
if math.isclose(fsize, s, rel_tol=0.05):
|
||||
file_info_cache[l] = t
|
||||
already_had_image_count += 1
|
||||
continue
|
||||
else:
|
||||
print(f'Found a mismatched size on file: {filepath} Redownloading!')
|
||||
download_work_queue.put((l, filepath, t, s))
|
||||
else:
|
||||
# We have a download record, confirm the file actually exists on disk
|
||||
if os.path.exists(filepath):
|
||||
already_had_image_count += 1
|
||||
elif filepath.endswith(".fits"):
|
||||
filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file
|
||||
filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file
|
||||
if os.path.exists(filepath2) or os.path.exists(filepath3):
|
||||
already_had_image_count += 1
|
||||
else: # We could not find the file on disk, queue for redownload
|
||||
download_work_queue.put((l, filepath, t))
|
||||
|
||||
with open(file_database_path, 'w') as f:
|
||||
f.write(json.dumps(file_info_cache))
|
||||
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
|
||||
if fetch_interval > 0:
|
||||
print(f"Run complete!, sleeping for {fetch_interval} seconds.")
|
||||
with open(file_database_path, 'w') as f:
|
||||
f.write(json.dumps(file_info_cache))
|
||||
time.sleep(fetch_interval)
|
||||
else:
|
||||
print("Run complete!, exiting...")
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
print("Saving file database and shutting down.")
|
||||
except Empty:
|
||||
print("Work Complete, Shutting down...")
|
||||
except Exception as e:
|
||||
print(f"Unhandled Exception during run: {e}")
|
||||
print("Shutting down")
|
||||
|
||||
with open(file_database_path, 'w') as f:
|
||||
f.write(json.dumps(file_info_cache))
|
||||
|
||||
for _ in range(nfetchworkers):
|
||||
try:
|
||||
query_work_queue.put(None, timeout=5.0)
|
||||
except:
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
|
||||
|
||||
for _ in range(ndownloadworkers):
|
||||
try:
|
||||
download_work_queue.put(None, timeout=5.0)
|
||||
except:
|
||||
break
|
||||
|
||||
for w in workers:
|
||||
import os
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import re
|
||||
import time
|
||||
import random
|
||||
import datetime
|
||||
import json
|
||||
from threading import Thread
|
||||
import queue
|
||||
import math
|
||||
from functools import partial
|
||||
from queue import Empty
|
||||
|
||||
import tqdm
|
||||
|
||||
directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/"
|
||||
stored_images_dir = os.path.abspath(os.path.join("..", "Data"))
|
||||
# directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/ephe-l2-orb1m/2017/12/"
|
||||
# stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16/l2/data/ephe-l2-orb1m/2017/12/"
|
||||
ignore_folder_names = ["Parent Directory", "l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
|
||||
file_database_path = os.path.abspath(os.path.join("..", "file_database.json"))
|
||||
fetch_interval = 0 # 60*60
|
||||
nfetchworkers = 2 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
|
||||
ndownloadworkers = 3 # Be nice to the servers, this value is how many threads will be downloading files at the same time
|
||||
randomize_order = False
|
||||
|
||||
links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace
|
||||
times_regex_pattern = r'(?<=<td align="right">)(\d{4}-\d{2}-\d{2} \d{2}:\d{2})(?= )' # Find timestamps in UTC in the format YYYY-MM-DD HH:mm
|
||||
sizes_regex_pattern = r'(?<=\d{4}-\d{2}-\d{2} \d{2}:\d{2} <\/td><td align="right">)([ .\d]{1,3})([KMG]|- |0 | )(?=<\/td>)' # Find file size markers in one of many possible formats
|
||||
links_matcher = re.compile(links_regex_pattern)
|
||||
times_matcher = re.compile(times_regex_pattern)
|
||||
sizes_matcher = re.compile(sizes_regex_pattern)
|
||||
|
||||
def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, randomize_order = True):
|
||||
while True:
|
||||
job = query_work_queue.get()
|
||||
if job == None:
|
||||
return
|
||||
url= job
|
||||
|
||||
attempts = 0
|
||||
html_content = None
|
||||
while attempts < attempt_count:
|
||||
try:
|
||||
with urllib.request.urlopen(url) as response:
|
||||
html_content = response.read().decode('utf-8')
|
||||
break
|
||||
except Exception as e:
|
||||
tqdm.tqdm.write(f"Exception while fetching links: {e}")
|
||||
time.sleep(1 + random.random())
|
||||
attempts += 1
|
||||
if (attempt_count > 1) and (attempts == attempt_count):
|
||||
tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
|
||||
|
||||
if html_content is None: # We failed to fetch the link for some reason, continue to the next job
|
||||
continue
|
||||
else:
|
||||
links = links_matcher.findall(html_content)
|
||||
times = times_matcher.findall(html_content)
|
||||
sizes = sizes_matcher.findall(html_content)
|
||||
for i in range(len(times)):
|
||||
dt = datetime.datetime.strptime(times[i].strip(), "%Y-%m-%d %H:%M")
|
||||
dt.replace(tzinfo=datetime.timezone.utc)
|
||||
times[i] = time.mktime(dt.timetuple())
|
||||
for i in range(len(sizes)):
|
||||
match sizes[i][1].strip():
|
||||
case '-':
|
||||
sizes[i] = 0
|
||||
case 'K':
|
||||
sizes[i] = int(float(sizes[i][0].strip()) * 1024)
|
||||
case 'M':
|
||||
sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024)
|
||||
case 'G':
|
||||
sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024 * 1024)
|
||||
case "0":
|
||||
sizes[i] = 0
|
||||
case "":
|
||||
sizes[i] = int(float(sizes[i][0].strip()))
|
||||
case _:
|
||||
raise(ValueError(f"Unexpected symbol while parsing links page: {_}"))
|
||||
|
||||
|
||||
if (len(links) != len(times)) or (len(times) != len(sizes)):
|
||||
raise(ValueError("Links parsing error, mismatched numbers of links, times, or sizes!"))
|
||||
|
||||
results = list(zip(links, times, sizes))
|
||||
if randomize_order:
|
||||
random.shuffle(results)
|
||||
for _link, _time, _size in results:
|
||||
if _link.endswith("/"):
|
||||
if _link.split(r"/")[-2] in ignore_folder_names:
|
||||
continue
|
||||
else:
|
||||
# print(f"Queueing folder: {urllib.parse.urljoin(url,_link)}")
|
||||
query_work_queue.put(urllib.parse.urljoin(url,_link))
|
||||
else:
|
||||
# print(f"Queueing file: {urllib.parse.urljoin(url,_link)}")
|
||||
query_result_queue.put((urllib.parse.urljoin(url,_link), _time, _size))
|
||||
|
||||
|
||||
# Fetch file from url and store it to path, retrying on failure
|
||||
def file_download_worker(download_work_queue, download_result_queue, attempt_count = 2):
|
||||
while True:
|
||||
job = download_work_queue.get()
|
||||
if job == None:
|
||||
return
|
||||
url, path, t, s = job
|
||||
|
||||
# print(f"Next job: {url}, {path}, {t}, {s}")
|
||||
attempts = 0
|
||||
while attempts < attempt_count:
|
||||
try:
|
||||
starttime = time.time()
|
||||
req = urllib.request.Request(url, data=None)
|
||||
image_data = urllib.request.urlopen(req, timeout=10.0).read()
|
||||
endtime = time.time()
|
||||
tqdm.tqdm.write(f"Downloaded {url} in {endtime-starttime:0.2f} s | {len(image_data)/1024/1024/(endtime-starttime):0.2f} MB/s")
|
||||
if math.isclose(len(image_data), s, rel_tol=0.05):
|
||||
os.makedirs(os.path.split(path)[0], exist_ok=True)
|
||||
open(path, 'wb').write(image_data)
|
||||
download_result_queue.put((True, url, t))
|
||||
break
|
||||
else:
|
||||
raise ValueError("Downloaded file is the wrong size!")
|
||||
except Exception as e:
|
||||
if hasattr(e, "code") and e.code == 404: # This is expected if the file has been removed from the site (at least for swpc.noaa.gov)
|
||||
attempts += attempt_count
|
||||
elif (0 < attempts) and (attempts < attempt_count):
|
||||
# tqdm.tqdm.write(f"\nA problem occurred on file: {url} | {e}")
|
||||
time.sleep(1 + random.random())
|
||||
attempts += 1
|
||||
if (attempt_count > 1) and (attempts == attempt_count):
|
||||
tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
|
||||
tqdm.tqdm.write(f"Exception: {e}")
|
||||
download_result_queue.put((False, url, t))
|
||||
break
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_info_cache = {}
|
||||
try:
|
||||
print(f"Attempting to load file records from cache: {file_database_path}")
|
||||
with open(file_database_path, 'r') as f:
|
||||
file_info_cache = json.loads(f.read())
|
||||
print(f"File records loaded from cache: {len(file_info_cache)} records found.")
|
||||
except Exception as e:
|
||||
print(f"Load failed, starting with empty cache")
|
||||
file_info_cache = {}
|
||||
|
||||
query_work_queue = queue.Queue()
|
||||
query_result_queue = queue.Queue()
|
||||
download_work_queue = queue.Queue(maxsize=ndownloadworkers)
|
||||
download_result_queue = queue.Queue(maxsize=ndownloadworkers)
|
||||
workers = []
|
||||
for _ in range(nfetchworkers):
|
||||
t = Thread(target=find_links_worker, args=(query_work_queue, query_result_queue, 3, randomize_order), daemon=True)
|
||||
t.start()
|
||||
workers.append(t)
|
||||
for _ in range(ndownloadworkers):
|
||||
t = Thread(target=file_download_worker, args=(download_work_queue, download_result_queue), daemon=True)
|
||||
t.start()
|
||||
workers.append(t)
|
||||
|
||||
try:
|
||||
while True:
|
||||
fetched_image_count = 0
|
||||
already_had_image_count = 0
|
||||
failed_image_count = 0
|
||||
urllen = len(directory_url)
|
||||
|
||||
query_work_queue.put(directory_url)
|
||||
for l, t, s in tqdm.tqdm(iter(partial(query_result_queue.get, timeout=30.0), None), desc="Downloading files"):
|
||||
# Collect completed jobs and record completion status
|
||||
while True:
|
||||
try:
|
||||
r_success, r_url, r_t = download_result_queue.get_nowait()
|
||||
if r_success:
|
||||
fetched_image_count += 1
|
||||
file_info_cache[r_url] = r_t
|
||||
else:
|
||||
failed_image_count += 1
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
file_portion_of_link = l[urllen:]
|
||||
filepath = os.path.join(stored_images_dir, file_portion_of_link.replace("/", os.sep))
|
||||
# If we dont have the file or the file at the link is newer than the one we previously fetched
|
||||
if (not (l in file_info_cache)) or t > file_info_cache[l]:
|
||||
if filepath.endswith(".fits"):
|
||||
filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file
|
||||
filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file
|
||||
if os.path.exists(filepath2) or os.path.exists(filepath3): # If the unfiltered filename exists, that case will be handled in the alternative code path in if os.path.exists(filepath):
|
||||
if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
|
||||
try:
|
||||
os.remove(filepath2)
|
||||
os.remove(filepath3)
|
||||
except:
|
||||
pass
|
||||
else: # If we have no record of this file, update the file info cache and don't redownload
|
||||
file_info_cache[l] = t
|
||||
already_had_image_count += 1
|
||||
continue
|
||||
if os.path.exists(filepath):
|
||||
if l in file_info_cache: # If we have record of this file, it must be out of date, delete it and download the new version.
|
||||
if os.path.exists(filepath):
|
||||
os.remove(filepath)
|
||||
else: # If we have no record of this file, update the file info cache and don't redownload if fsize is right
|
||||
fsize = os.path.getsize(filepath)
|
||||
if math.isclose(fsize, s, rel_tol=0.05):
|
||||
file_info_cache[l] = t
|
||||
already_had_image_count += 1
|
||||
continue
|
||||
else:
|
||||
tqdm.tqdm.write(f'Found a mismatched size on file: {filepath} Redownloading!')
|
||||
download_work_queue.put((l, filepath, t, s))
|
||||
else:
|
||||
# We have a download record, confirm the file actually exists on disk
|
||||
if os.path.exists(filepath):
|
||||
already_had_image_count += 1
|
||||
elif filepath.endswith(".fits"):
|
||||
filepath2 = filepath.split(".fits")[0] + "_f.fits" # also look for the filtered version of the file
|
||||
filepath3 = filepath.split(".fits")[0] + "_e.fits" # also look for the error version of the file
|
||||
if os.path.exists(filepath2) or os.path.exists(filepath3):
|
||||
already_had_image_count += 1
|
||||
else: # We could not find the file on disk, queue for redownload
|
||||
download_work_queue.put((l, filepath, t))
|
||||
|
||||
with open(file_database_path, 'w') as f:
|
||||
f.write(json.dumps(file_info_cache))
|
||||
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
|
||||
if fetch_interval > 0:
|
||||
print(f"Run complete!, sleeping for {fetch_interval} seconds.")
|
||||
with open(file_database_path, 'w') as f:
|
||||
f.write(json.dumps(file_info_cache))
|
||||
time.sleep(fetch_interval)
|
||||
else:
|
||||
print("Run complete!, exiting...")
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
print("Saving file database and shutting down.")
|
||||
except Empty:
|
||||
print("Work Complete, Shutting down...")
|
||||
except Exception as e:
|
||||
print(f"Unhandled Exception during run: {e}")
|
||||
print("Shutting down")
|
||||
|
||||
for _ in range(nfetchworkers):
|
||||
try:
|
||||
query_work_queue.put(None)
|
||||
except:
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
|
||||
|
||||
while len(download_work_queue.queue) > 0:
|
||||
print(f"Waiting for {len(download_work_queue.queue)} downloads in queue...")
|
||||
time.sleep(1)
|
||||
|
||||
for _ in range(ndownloadworkers):
|
||||
try:
|
||||
download_work_queue.put(None, timeout=5.0)
|
||||
except:
|
||||
break
|
||||
|
||||
with open(file_database_path, 'w') as f:
|
||||
f.write(json.dumps(file_info_cache))
|
||||
|
||||
print("Waiting for workers to shutdown...")
|
||||
|
||||
for w in workers:
|
||||
w.join(5.0)
|
||||
8
requirements.txt
Normal file
8
requirements.txt
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
astropy
|
||||
scikit-image
|
||||
tqdm
|
||||
numpy
|
||||
opencv-python
|
||||
palettable
|
||||
matplotlib
|
||||
pillow
|
||||
Loading…
Add table
Reference in a new issue