jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Updated fetcher and added .vscode folder to gitignore

Browse source
This commit is contained in:
Jeremy Karst 2026-08-28 01:39:18 -04:00
parent 756270cd1b
commit 830a7eb469
6 changed files with 1213 additions and 1178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
.vscode

16
ffmpeg_video.py
View file

@ -10,12 +10,18 @@ from sortedcontainers import SortedDict
import tqdm
from PIL import Image
path_to_images = r"..\composite\goes18"
output_file = r"..\goes18_2023.mp4"
interp_file = r"..\goes18_2023_interp.mp4"
years = [2024]
sources = ["goes16", "goes18"]
ffmpeg_path = r"..\ffmpeg.exe"
starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
for year in years:
for source in sources:
path_to_images = f"..\\composite\\{source}"
output_file = f"..\\{source}_{year}_nofilt.mp4"
interp_file = f"..\\{source}_{year}_interp_nofilt.mp4"
starttime = calendar.timegm(datetime.datetime(year, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = calendar.timegm(datetime.datetime(year+1, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
min_file_size = 350000 # Detect and remove corrupted files by filtering by file size
max_file_size = 450000
encoding_crf = 16

4
filter_FITS.py
View file

@ -52,7 +52,7 @@ avg_radius = (valid_radii[0] + valid_radii[1]) // 2
max_radius_error = 80
ratio_above_thresh_max = 0.4
ratio_above_thresh_min = 0.1
ratio_above_thresh_min = 0.07
def filter_fits(work_queue):
lowpriority()
@ -211,7 +211,7 @@ if __name__ == "__main__":
starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
reprocess_errors = False
reprocess_errors = True
nworkers = 16
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"

9
merger_FITS.py
View file

@ -439,9 +439,14 @@ if __name__ == "__main__":
stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$"
nworkers = 16
max_time_gap = 3
fill_missing_data = True
fill_missing_data = False
ignore_errors = True
if ignore_errors:
regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*.fits$"
file_prefixes = ["dr_suvi-l2-ci" + n[:-1] for n in image_names]
@ -556,7 +561,7 @@ if __name__ == "__main__":
else: # We have a complete file set, update the last_good_files
last_good_files = files_this_timestamp
last_good_file_times = [timestamp for _ in last_good_files]
work_queue.put((files_this_timestamp, timestamp, processed_images_dir, False))
work_queue.put((files_this_timestamp, timestamp, processed_images_dir, ignore_errors))
except KeyboardInterrupt:
print("Finishing current jobs and exiting")

53
puller_fits.py
View file

@ -15,14 +15,14 @@ from queue import Empty
import tqdm
directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/"
stored_images_dir = r"..\Data"
stored_images_dir = os.path.abspath(os.path.join("..", "Data"))
# directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/ephe-l2-orb1m/2017/12/"
# stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16/l2/data/ephe-l2-orb1m/2017/12/"
ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
file_database_path = r"..\file_database.json"
ignore_folder_names = ["Parent Directory", "l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
file_database_path = os.path.abspath(os.path.join("..", "file_database.json"))
fetch_interval = 0 # 60*60
nfetchworkers = 10 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
ndownloadworkers = 4 # Be nice to the servers, this value is how many threads will be downloading files at the same time
nfetchworkers = 2 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
ndownloadworkers = 3 # Be nice to the servers, this value is how many threads will be downloading files at the same time
randomize_order = False
links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace
@ -47,13 +47,15 @@ def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, r
html_content = response.read().decode('utf-8')
break
except Exception as e:
print(f"Exception while fetching links: {e}")
tqdm.tqdm.write(f"Exception while fetching links: {e}")
time.sleep(1 + random.random())
attempts += 1
if (attempt_count > 1) and (attempts == attempt_count):
print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
return
tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
if html_content is None: # We failed to fetch the link for some reason, continue to the next job
continue
else:
links = links_matcher.findall(html_content)
times = times_matcher.findall(html_content)
sizes = sizes_matcher.findall(html_content)
@ -90,8 +92,10 @@ def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, r
if _link.split(r"/")[-2] in ignore_folder_names:
continue
else:
# print(f"Queueing folder: {urllib.parse.urljoin(url,_link)}")
query_work_queue.put(urllib.parse.urljoin(url,_link))
else:
# print(f"Queueing file: {urllib.parse.urljoin(url,_link)}")
query_result_queue.put((urllib.parse.urljoin(url,_link), _time, _size))
@ -103,11 +107,15 @@ def file_download_worker(download_work_queue, download_result_queue, attempt_cou
return
url, path, t, s = job
# print(f"Next job: {url}, {path}, {t}, {s}")
attempts = 0
while attempts < attempt_count:
try:
starttime = time.time()
req = urllib.request.Request(url, data=None)
image_data = urllib.request.urlopen(req).read()
image_data = urllib.request.urlopen(req, timeout=10.0).read()
endtime = time.time()
tqdm.tqdm.write(f"Downloaded {url} in {endtime-starttime:0.2f} s | {len(image_data)/1024/1024/(endtime-starttime):0.2f} MB/s")
if math.isclose(len(image_data), s, rel_tol=0.05):
os.makedirs(os.path.split(path)[0], exist_ok=True)
open(path, 'wb').write(image_data)
@ -118,12 +126,13 @@ def file_download_worker(download_work_queue, download_result_queue, attempt_cou
except Exception as e:
if hasattr(e, "code") and e.code == 404: # This is expected if the file has been removed from the site (at least for swpc.noaa.gov)
attempts += attempt_count
elif attempts == 0:
print(f"\nA problem occurred on image: {url} | {e}")
elif (0 < attempts) and (attempts < attempt_count):
# tqdm.tqdm.write(f"\nA problem occurred on file: {url} | {e}")
time.sleep(1 + random.random())
attempts += 1
if (attempt_count > 1) and (attempts == attempt_count):
print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
tqdm.tqdm.write(f"Exception: {e}")
download_result_queue.put((False, url, t))
break
@ -141,8 +150,8 @@ if __name__ == "__main__":
query_work_queue = queue.Queue()
query_result_queue = queue.Queue()
download_work_queue = queue.Queue()
download_result_queue = queue.Queue()
download_work_queue = queue.Queue(maxsize=ndownloadworkers)
download_result_queue = queue.Queue(maxsize=ndownloadworkers)
workers = []
for _ in range(nfetchworkers):
t = Thread(target=find_links_worker, args=(query_work_queue, query_result_queue, 3, randomize_order), daemon=True)
@ -203,7 +212,7 @@ if __name__ == "__main__":
already_had_image_count += 1
continue
else:
print(f'Found a mismatched size on file: {filepath} Redownloading!')
tqdm.tqdm.write(f'Found a mismatched size on file: {filepath} Redownloading!')
download_work_queue.put((l, filepath, t, s))
else:
# We have a download record, confirm the file actually exists on disk
@ -236,23 +245,29 @@ if __name__ == "__main__":
print(f"Unhandled Exception during run: {e}")
print("Shutting down")
with open(file_database_path, 'w') as f:
f.write(json.dumps(file_info_cache))
for _ in range(nfetchworkers):
try:
query_work_queue.put(None, timeout=5.0)
query_work_queue.put(None)
except:
break
time.sleep(1)
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
while len(download_work_queue.queue) > 0:
print(f"Waiting for {len(download_work_queue.queue)} downloads in queue...")
time.sleep(1)
for _ in range(ndownloadworkers):
try:
download_work_queue.put(None, timeout=5.0)
except:
break
with open(file_database_path, 'w') as f:
f.write(json.dumps(file_info_cache))
print("Waiting for workers to shutdown...")
for w in workers:
w.join(5.0)

8
requirements.txt Normal file
View file

@ -0,0 +1,8 @@
astropy
scikit-image
tqdm
numpy
opencv-python
palettable
matplotlib
pillow