jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixed bug in puller causing bailout during connection error

Browse source
This commit is contained in:
Jeremy Karst 2024-05-18 19:06:50 -04:00
parent a47c8b95d0
commit 133a4f563c
1 changed files with 48 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
puller_fits.py
View file

@ -15,55 +15,55 @@ directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-sat
stored_images_dir = r"..\Data" stored_images_dir = r"..\Data"
# directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/suvi-l2-ci094/" # directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/suvi-l2-ci094/"
# stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16\l2\suvi-l2-ci094" # stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16\l2\suvi-l2-ci094"
ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022"] ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020"]
file_database_path = r"..\file_database.json" file_database_path = r"..\file_database.json"
fetch_interval = 60*60 fetch_interval = 0 # 60*60
nworkers = 16 nworkers = 8
randomize_order = True randomize_order = False
def recursive_find_links(url, attempt_count = 3, randomize_order = True): def recursive_find_links(url, attempt_count = 3, randomize_order = True):
links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace
times_regex_pattern = r'(?<=<td align="right">)(\d{4}-\d{2}-\d{2} \d{2}:\d{2})(?= )' # Find timestamps in UTC in the format YYYY-MM-DD HH:mm times_regex_pattern = r'(?<=<td align="right">)(\d{4}-\d{2}-\d{2} \d{2}:\d{2})(?= )' # Find timestamps in UTC in the format YYYY-MM-DD HH:mm
with urllib.request.urlopen(url) as response: attempts = 0
attempts = 0 html_content = None
html_content = None while attempts < attempt_count:
while attempts < attempt_count: try:
try: with urllib.request.urlopen(url) as response:
html_content = response.read().decode('utf-8') html_content = response.read().decode('utf-8')
break break
except Exception as e: except Exception as e:
print(f"Exception while fetching links: {e}") print(f"Exception while fetching links: {e}")
time.sleep(1 + random.random()) time.sleep(1 + random.random())
attempts += 1 attempts += 1
if (attempt_count > 1) and (attempts == attempt_count): if (attempt_count > 1) and (attempts == attempt_count):
print(f'\nAfter {attempt_count} retries, could not fetch: {url}') print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
return return
links = re.findall(links_regex_pattern, html_content) links = re.findall(links_regex_pattern, html_content)
times = re.findall(times_regex_pattern, html_content) times = re.findall(times_regex_pattern, html_content)
for i in range(len(times)): for i in range(len(times)):
dt = datetime.datetime.strptime(times[i].strip(), "%Y-%m-%d %H:%M") dt = datetime.datetime.strptime(times[i].strip(), "%Y-%m-%d %H:%M")
dt.replace(tzinfo=datetime.timezone.utc) dt.replace(tzinfo=datetime.timezone.utc)
times[i] = time.mktime(dt.timetuple()) times[i] = time.mktime(dt.timetuple())
if len(links) - 1 == len(times): if len(links) - 1 == len(times):
links = links[1:] links = links[1:]
elif len(links) == len(times): elif len(links) == len(times):
pass pass
else: else:
raise(ValueError) raise(ValueError)
results = list(zip(links, times)) results = list(zip(links, times))
if randomize_order: if randomize_order:
random.shuffle(results) random.shuffle(results)
for _link, _time in results: for _link, _time in results:
if _link.endswith("/"): if _link.endswith("/"):
if _link.split(r"/")[-2] in ignore_folder_names: if _link.split(r"/")[-2] in ignore_folder_names:
continue continue
else:
yield from recursive_find_links(urllib.parse.urljoin(url,_link), randomize_order)
else: else:
yield urllib.parse.urljoin(url,_link), _time yield from recursive_find_links(urllib.parse.urljoin(url,_link), attempt_count, randomize_order)
else:
yield urllib.parse.urljoin(url,_link), _time
# Fetch image from url and store it to path, retrying on failure # Fetch image from url and store it to path, retrying on failure
@ -120,7 +120,7 @@ if __name__ == "__main__":
already_had_image_count = 0 already_had_image_count = 0
failed_image_count = 0 failed_image_count = 0
urllen = len(directory_url) urllen = len(directory_url)
for l, t in tqdm.tqdm(recursive_find_links(directory_url, randomize_order), desc="Downloading files"): for l, t in tqdm.tqdm(recursive_find_links(directory_url, randomize_order = randomize_order), desc="Downloading files"):
# Collect completed jobs and record completion status # Collect completed jobs and record completion status
while True: while True:
try: try:
@ -152,7 +152,14 @@ if __name__ == "__main__":
with open(file_database_path, 'w') as f: with open(file_database_path, 'w') as f:
f.write(json.dumps(file_info_cache)) f.write(json.dumps(file_info_cache))
print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}") print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
time.sleep(fetch_interval) if fetch_interval > 0:
print(f"Run complete!, sleeping for {fetch_interval} seconds.")
with open(file_database_path, 'w') as f:
f.write(json.dumps(file_info_cache))
time.sleep(fetch_interval)
else:
print("Run complete!, exiting...")
break
except KeyboardInterrupt: except KeyboardInterrupt:
print("Saving file database and shutting down.") print("Saving file database and shutting down.")