Updated fetcher and added .vscode folder to gitignore

.gitignore

@@ -0,0 +1 @@
+.vscode

ffmpeg_video.py

@@ -10,26 +10,32 @@ from sortedcontainers import SortedDict
 import tqdm
 from PIL import Image
 
-path_to_images = r"..\composite\goes18"
+years = [2024]
-output_file = r"..\goes18_2023.mp4"
+sources = ["goes16", "goes18"]
-interp_file = r"..\goes18_2023_interp.mp4"
 ffmpeg_path = r"..\ffmpeg.exe"
-starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-min_file_size = 350000 # Detect and remove corrupted files by filtering by file size
-max_file_size = 450000
-encoding_crf = 16
-max_frame_interp = 120
 
-# Set up ffmpeg to stream images from an input pipe
+for year in years:
-command_line = f'{ffmpeg_path} -y -f image2pipe -framerate 60 -i - -c:v libx264 -crf {encoding_crf} -preset veryfast {output_file}'
+    for source in sources:
-print(command_line)
+        path_to_images = f"..\\composite\\{source}"
-p = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+        output_file = f"..\\{source}_{year}_nofilt.mp4"
+        interp_file = f"..\\{source}_{year}_interp_nofilt.mp4"
 
-black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
+        starttime = calendar.timegm(datetime.datetime(year, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
+        stoptime = calendar.timegm(datetime.datetime(year+1, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
+        min_file_size = 350000 # Detect and remove corrupted files by filtering by file size
+        max_file_size = 450000
+        encoding_crf = 16
+        max_frame_interp = 120
 
-files_by_time = SortedDict()
+        # Set up ffmpeg to stream images from an input pipe
-for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and filtering files."):
+        command_line = f'{ffmpeg_path} -y -f image2pipe -framerate 60 -i - -c:v libx264 -crf {encoding_crf} -preset veryfast {output_file}'
+        print(command_line)
+        p = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+
+        black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
+
+        files_by_time = SortedDict()
+        for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and filtering files."):
             for f in files:
                 if f.endswith('.jpg'):
                     fpath = os.path.join(root, f)
@@ -39,13 +45,13 @@ for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and fi
                     if (fsize > min_file_size) and (fsize < max_file_size) and (ftime > starttime) and (ftime < stoptime):
                         files_by_time[ftime] = fpath
 
-difftimes = np.diff(files_by_time.keys())
+        difftimes = np.diff(files_by_time.keys())
-unique, counts = np.unique(difftimes, return_counts=True)
+        unique, counts = np.unique(difftimes, return_counts=True)
-interval = unique[0]
+        interval = unique[0]
-assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval
+        assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval
 
-prevtime = files_by_time.peekitem(0)[0] - interval
+        prevtime = files_by_time.peekitem(0)[0] - interval
-for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", total = len(files_by_time)):
+        for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", total = len(files_by_time)):
             framejump = (t - prevtime) // interval
             if framejump < max_frame_interp:
                 for i in range(framejump - 1): # Fill black frames to fill gaps, these will later be interpolated with ffmpeg
@@ -56,12 +62,12 @@ for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", tota
                 p.stdin.write(fh.read())
             prevtime = t
 
-p.stdin.close() # Close the ffmpeg input pipe
+        p.stdin.close() # Close the ffmpeg input pipe
-p.wait() # Wait for ffmpeg to finish encoding
+        p.wait() # Wait for ffmpeg to finish encoding
 
-command_line = f'{ffmpeg_path} -y -i {output_file} -vf blackframe=0,metadata=select:key=lavfi.blackframe.pblack:value=95:function=less,minterpolate=mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:me=fss -c:a copy -c:v libx264 -crf {encoding_crf} -preset veryfast {interp_file}'
+        command_line = f'{ffmpeg_path} -y -i {output_file} -vf blackframe=0,metadata=select:key=lavfi.blackframe.pblack:value=95:function=less,minterpolate=mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:me=fss -c:a copy -c:v libx264 -crf {encoding_crf} -preset veryfast {interp_file}'
-print(command_line)
+        print(command_line)
 
-pipe = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE).stdout
+        pipe = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE).stdout
-output = pipe.read().decode()
+        output = pipe.read().decode()
-pipe.close()
+        pipe.close()

filter_FITS.py

@@ -52,7 +52,7 @@ avg_radius = (valid_radii[0] + valid_radii[1]) // 2
 max_radius_error = 80
 
 ratio_above_thresh_max = 0.4
-ratio_above_thresh_min = 0.1
+ratio_above_thresh_min = 0.07
 
 def filter_fits(work_queue):
     lowpriority()
@@ -211,7 +211,7 @@ if __name__ == "__main__":
     starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
     stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
 
-    reprocess_errors = False
+    reprocess_errors = True
     nworkers = 16
 
     regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"

merger_FITS.py

@@ -439,9 +439,14 @@ if __name__ == "__main__":
     stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
 
     regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$"
+
     nworkers = 16
     max_time_gap = 3
-    fill_missing_data = True
+    fill_missing_data = False
+    ignore_errors = True
+
+    if ignore_errors:
+        regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*.fits$"
 
     file_prefixes = ["dr_suvi-l2-ci" + n[:-1] for n in image_names]
     
@@ -556,7 +561,7 @@ if __name__ == "__main__":
                 else: # We have a complete file set, update the last_good_files
                     last_good_files = files_this_timestamp
                     last_good_file_times = [timestamp for _ in last_good_files]
-                    work_queue.put((files_this_timestamp, timestamp, processed_images_dir, False))
+                    work_queue.put((files_this_timestamp, timestamp, processed_images_dir, ignore_errors))
 
     except KeyboardInterrupt:
         print("Finishing current jobs and exiting")

puller_fits.py

@@ -15,14 +15,14 @@ from queue import Empty
 import tqdm
 
 directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/"
-stored_images_dir = r"..\Data"
+stored_images_dir = os.path.abspath(os.path.join("..", "Data"))
 # directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/ephe-l2-orb1m/2017/12/"
 # stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16/l2/data/ephe-l2-orb1m/2017/12/"
-ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
+ignore_folder_names = ["Parent Directory", "l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
-file_database_path = r"..\file_database.json"
+file_database_path = os.path.abspath(os.path.join("..", "file_database.json"))
 fetch_interval = 0 # 60*60
-nfetchworkers = 10 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
+nfetchworkers = 2 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
-ndownloadworkers = 4 # Be nice to the servers, this value is how many threads will be downloading files at the same time
+ndownloadworkers = 3 # Be nice to the servers, this value is how many threads will be downloading files at the same time
 randomize_order = False
 
 links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace
@@ -47,13 +47,15 @@ def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, r
                     html_content = response.read().decode('utf-8')
                     break
             except Exception as e:
-                print(f"Exception while fetching links: {e}")
+                tqdm.tqdm.write(f"Exception while fetching links: {e}")
                 time.sleep(1 + random.random())
                 attempts += 1
             if (attempt_count > 1) and (attempts == attempt_count):
-                print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
+                tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
-                return
 
+        if html_content is None: # We failed to fetch the link for some reason, continue to the next job
+            continue
+        else:
             links = links_matcher.findall(html_content)
             times = times_matcher.findall(html_content)
             sizes = sizes_matcher.findall(html_content)
@@ -90,8 +92,10 @@ def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, r
                     if _link.split(r"/")[-2] in ignore_folder_names:
                         continue
                     else:
+                        # print(f"Queueing folder: {urllib.parse.urljoin(url,_link)}")
                         query_work_queue.put(urllib.parse.urljoin(url,_link))
                 else:
+                    # print(f"Queueing file: {urllib.parse.urljoin(url,_link)}")
                     query_result_queue.put((urllib.parse.urljoin(url,_link), _time, _size))
     
 
@@ -103,11 +107,15 @@ def file_download_worker(download_work_queue, download_result_queue, attempt_cou
             return
         url, path, t, s = job
 
+        # print(f"Next job: {url}, {path}, {t}, {s}")
         attempts = 0
         while attempts < attempt_count:
             try:
+                starttime = time.time()
                 req = urllib.request.Request(url, data=None)
-                image_data = urllib.request.urlopen(req).read()
+                image_data = urllib.request.urlopen(req, timeout=10.0).read()
+                endtime = time.time()
+                tqdm.tqdm.write(f"Downloaded {url} in {endtime-starttime:0.2f} s | {len(image_data)/1024/1024/(endtime-starttime):0.2f} MB/s")
                 if math.isclose(len(image_data), s, rel_tol=0.05):
                     os.makedirs(os.path.split(path)[0], exist_ok=True)
                     open(path, 'wb').write(image_data)
@@ -118,12 +126,13 @@ def file_download_worker(download_work_queue, download_result_queue, attempt_cou
             except Exception as e:
                 if hasattr(e, "code") and e.code == 404: # This is expected if the file has been removed from the site (at least for swpc.noaa.gov)
                     attempts += attempt_count
-                elif attempts == 0:
+                elif (0 < attempts) and (attempts < attempt_count):
-                    print(f"\nA problem occurred on image: {url} | {e}")
+                    # tqdm.tqdm.write(f"\nA problem occurred on file: {url} | {e}")
                     time.sleep(1 + random.random())
                     attempts += 1
                 if (attempt_count > 1) and (attempts == attempt_count):
-                print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
+                    tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
+                    tqdm.tqdm.write(f"Exception: {e}")
                     download_result_queue.put((False, url, t))
                     break
 
@@ -141,8 +150,8 @@ if __name__ == "__main__":
 
     query_work_queue = queue.Queue()
     query_result_queue = queue.Queue()
-    download_work_queue = queue.Queue()
+    download_work_queue = queue.Queue(maxsize=ndownloadworkers)
-    download_result_queue = queue.Queue()
+    download_result_queue = queue.Queue(maxsize=ndownloadworkers)
     workers = []
     for _ in range(nfetchworkers):
         t = Thread(target=find_links_worker, args=(query_work_queue, query_result_queue, 3, randomize_order), daemon=True)
@@ -203,7 +212,7 @@ if __name__ == "__main__":
                                 already_had_image_count += 1
                                 continue
                             else:
-                                print(f'Found a mismatched size on file: {filepath}  Redownloading!')
+                                tqdm.tqdm.write(f'Found a mismatched size on file: {filepath}  Redownloading!')
                     download_work_queue.put((l, filepath, t, s))
                 else:
                     # We have a download record, confirm the file actually exists on disk
@@ -236,23 +245,29 @@ if __name__ == "__main__":
         print(f"Unhandled Exception during run: {e}")
         print("Shutting down")
 
-    with open(file_database_path, 'w') as f:
-        f.write(json.dumps(file_info_cache))
-
     for _ in range(nfetchworkers):
         try:
-            query_work_queue.put(None, timeout=5.0)
+            query_work_queue.put(None)
         except:
             break
 
     time.sleep(1)
     print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
 
+    while len(download_work_queue.queue) > 0:
+        print(f"Waiting for {len(download_work_queue.queue)} downloads in queue...")
+        time.sleep(1)
+
     for _ in range(ndownloadworkers):
         try:
             download_work_queue.put(None, timeout=5.0)
         except:
             break
 
+    with open(file_database_path, 'w') as f:
+        f.write(json.dumps(file_info_cache))
+
+    print("Waiting for workers to shutdown...")
+
     for w in workers:
         w.join(5.0)

requirements.txt

@@ -0,0 +1,8 @@
+astropy
+scikit-image
+tqdm
+numpy
+opencv-python
+palettable
+matplotlib
+pillow