Updated fetcher and added .vscode folder to gitignore

.gitignore

@@ -0,0 +1 @@
+.vscode

ffmpeg_video.py

@@ -10,58 +10,64 @@ from sortedcontainers import SortedDict
 import tqdm
 from PIL import Image
 
-path_to_images = r"..\composite\goes18"
+years = [2024]
-output_file = r"..\goes18_2023.mp4"
+sources = ["goes16", "goes18"]
-interp_file = r"..\goes18_2023_interp.mp4"
 ffmpeg_path = r"..\ffmpeg.exe"
-starttime = calendar.timegm(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-stoptime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
-min_file_size = 350000 # Detect and remove corrupted files by filtering by file size
-max_file_size = 450000
-encoding_crf = 16
-max_frame_interp = 120
 
-# Set up ffmpeg to stream images from an input pipe
+for year in years:
-command_line = f'{ffmpeg_path} -y -f image2pipe -framerate 60 -i - -c:v libx264 -crf {encoding_crf} -preset veryfast {output_file}'
+    for source in sources:
-print(command_line)
+        path_to_images = f"..\\composite\\{source}"
-p = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+        output_file = f"..\\{source}_{year}_nofilt.mp4"
+        interp_file = f"..\\{source}_{year}_interp_nofilt.mp4"
 
-black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
+        starttime = calendar.timegm(datetime.datetime(year, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
+        stoptime = calendar.timegm(datetime.datetime(year+1, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
+        min_file_size = 350000 # Detect and remove corrupted files by filtering by file size
+        max_file_size = 450000
+        encoding_crf = 16
+        max_frame_interp = 120
 
-files_by_time = SortedDict()
+        # Set up ffmpeg to stream images from an input pipe
-for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and filtering files."):
+        command_line = f'{ffmpeg_path} -y -f image2pipe -framerate 60 -i - -c:v libx264 -crf {encoding_crf} -preset veryfast {output_file}'
-    for f in files:
+        print(command_line)
-        if f.endswith('.jpg'):
+        p = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
-            fpath = os.path.join(root, f)
-            fsize = os.path.getsize(fpath)
-            time_chunk = f.replace('-', '.').replace('_', '.').split('.')[1]
-            ftime = int(time_chunk)
-            if (fsize > min_file_size) and (fsize < max_file_size) and (ftime > starttime) and (ftime < stoptime):
-                files_by_time[ftime] = fpath
 
-difftimes = np.diff(files_by_time.keys())
+        black_image = Image.fromarray(np.zeros((1080, 1920, 3), dtype='uint8'))
-unique, counts = np.unique(difftimes, return_counts=True)
-interval = unique[0]
-assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval
 
-prevtime = files_by_time.peekitem(0)[0] - interval
+        files_by_time = SortedDict()
-for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", total = len(files_by_time)):
+        for root, dirs, files in tqdm.tqdm(os.walk(path_to_images), desc="Finding and filtering files."):
-    framejump = (t - prevtime) // interval
+            for f in files:
-    if framejump < max_frame_interp:
+                if f.endswith('.jpg'):
-        for i in range(framejump - 1): # Fill black frames to fill gaps, these will later be interpolated with ffmpeg
+                    fpath = os.path.join(root, f)
-            black_image.save(p.stdin, 'jpeg', quality = 95)
+                    fsize = os.path.getsize(fpath)
-    else: # Skip past intervals that are too large to fill reasonably
+                    time_chunk = f.replace('-', '.').replace('_', '.').split('.')[1]
-        print(f"Detected a frame gap of: {framejump}!")
+                    ftime = int(time_chunk)
-    with open(f, 'rb') as fh:
+                    if (fsize > min_file_size) and (fsize < max_file_size) and (ftime > starttime) and (ftime < stoptime):
-        p.stdin.write(fh.read())
+                        files_by_time[ftime] = fpath
-    prevtime = t
 
-p.stdin.close() # Close the ffmpeg input pipe
+        difftimes = np.diff(files_by_time.keys())
-p.wait() # Wait for ffmpeg to finish encoding
+        unique, counts = np.unique(difftimes, return_counts=True)
+        interval = unique[0]
+        assert np.sum((unique % interval) > 0) == 0 # Ensure all our timestamps align perfectly with our interval
 
-command_line = f'{ffmpeg_path} -y -i {output_file} -vf blackframe=0,metadata=select:key=lavfi.blackframe.pblack:value=95:function=less,minterpolate=mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:me=fss -c:a copy -c:v libx264 -crf {encoding_crf} -preset veryfast {interp_file}'
+        prevtime = files_by_time.peekitem(0)[0] - interval
-print(command_line)
+        for t, f in tqdm.tqdm(files_by_time.items(), desc="FFMPEG encoding images", total = len(files_by_time)):
+            framejump = (t - prevtime) // interval
+            if framejump < max_frame_interp:
+                for i in range(framejump - 1): # Fill black frames to fill gaps, these will later be interpolated with ffmpeg
+                    black_image.save(p.stdin, 'jpeg', quality = 95)
+            else: # Skip past intervals that are too large to fill reasonably
+                print(f"Detected a frame gap of: {framejump}!")
+            with open(f, 'rb') as fh:
+                p.stdin.write(fh.read())
+            prevtime = t
 
-pipe = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE).stdout
+        p.stdin.close() # Close the ffmpeg input pipe
-output = pipe.read().decode()
+        p.wait() # Wait for ffmpeg to finish encoding
-pipe.close()
+
+        command_line = f'{ffmpeg_path} -y -i {output_file} -vf blackframe=0,metadata=select:key=lavfi.blackframe.pblack:value=95:function=less,minterpolate=mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:me=fss -c:a copy -c:v libx264 -crf {encoding_crf} -preset veryfast {interp_file}'
+        print(command_line)
+
+        pipe = subprocess.Popen(command_line, shell=True, stdout=subprocess.PIPE).stdout
+        output = pipe.read().decode()
+        pipe.close()

filter_FITS.py

@@ -52,7 +52,7 @@ avg_radius = (valid_radii[0] + valid_radii[1]) // 2
 max_radius_error = 80
 
 ratio_above_thresh_max = 0.4
-ratio_above_thresh_min = 0.1
+ratio_above_thresh_min = 0.07
 
 def filter_fits(work_queue):
     lowpriority()
@@ -211,7 +211,7 @@ if __name__ == "__main__":
     starttime = calendar.timegm(datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
     stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
 
-    reprocess_errors = False
+    reprocess_errors = True
     nworkers = 16
 
     regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\.fits$"

merger_FITS.py

@@ -439,9 +439,14 @@ if __name__ == "__main__":
     stoptime = calendar.timegm(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timetuple())
 
     regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*\_f.fits$"
+
     nworkers = 16
     max_time_gap = 3
-    fill_missing_data = True
+    fill_missing_data = False
+    ignore_errors = True
+
+    if ignore_errors:
+        regex_filename = r"dr_suvi-l2-ci\d{3}_g(16|18)_s\S*.fits$"
 
     file_prefixes = ["dr_suvi-l2-ci" + n[:-1] for n in image_names]
     
@@ -556,7 +561,7 @@ if __name__ == "__main__":
                 else: # We have a complete file set, update the last_good_files
                     last_good_files = files_this_timestamp
                     last_good_file_times = [timestamp for _ in last_good_files]
-                    work_queue.put((files_this_timestamp, timestamp, processed_images_dir, False))
+                    work_queue.put((files_this_timestamp, timestamp, processed_images_dir, ignore_errors))
 
     except KeyboardInterrupt:
         print("Finishing current jobs and exiting")

puller_fits.py

@@ -15,14 +15,14 @@ from queue import Empty
 import tqdm
 
 directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/"
-stored_images_dir = r"..\Data"
+stored_images_dir = os.path.abspath(os.path.join("..", "Data"))
 # directory_url = r"https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/goes16/l2/data/ephe-l2-orb1m/2017/12/"
 # stored_images_dir = r"Z:\NOAA GOES Data\Data\goes16/l2/data/ephe-l2-orb1m/2017/12/"
-ignore_folder_names = ["l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
+ignore_folder_names = ["Parent Directory", "l1b", "goes17", "2017", "2018", "2019", "2020", "2021", "2022", "2023"]
-file_database_path = r"..\file_database.json"
+file_database_path = os.path.abspath(os.path.join("..", "file_database.json"))
 fetch_interval = 0 # 60*60
-nfetchworkers = 10 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
+nfetchworkers = 2 # Be nice to the servers, this value is how many threads will be asking for links and file info at the same time
-ndownloadworkers = 4 # Be nice to the servers, this value is how many threads will be downloading files at the same time
+ndownloadworkers = 3 # Be nice to the servers, this value is how many threads will be downloading files at the same time
 randomize_order = False
 
 links_regex_pattern = r'(?<=<a href=")([^ ?:]*)(?=">.*\d{4}-\d{2}-\d{2} \d{2}:\d{2})' # Find href links that do not contain question marks or whitespace
@@ -47,52 +47,56 @@ def find_links_worker(query_work_queue, query_result_queue, attempt_count = 3, r
                     html_content = response.read().decode('utf-8')
                     break
             except Exception as e:
-                print(f"Exception while fetching links: {e}")
+                tqdm.tqdm.write(f"Exception while fetching links: {e}")
                 time.sleep(1 + random.random())
                 attempts += 1
             if (attempt_count > 1) and (attempts == attempt_count):
-                print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
+                tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
-                return
 
-        links = links_matcher.findall(html_content)
+        if html_content is None: # We failed to fetch the link for some reason, continue to the next job
-        times = times_matcher.findall(html_content)
+            continue
-        sizes = sizes_matcher.findall(html_content)
+        else:
-        for i in range(len(times)):
+            links = links_matcher.findall(html_content)
-            dt = datetime.datetime.strptime(times[i].strip(), "%Y-%m-%d %H:%M")
+            times = times_matcher.findall(html_content)
-            dt.replace(tzinfo=datetime.timezone.utc)
+            sizes = sizes_matcher.findall(html_content)
-            times[i] = time.mktime(dt.timetuple())
+            for i in range(len(times)):
-        for i in range(len(sizes)):
+                dt = datetime.datetime.strptime(times[i].strip(), "%Y-%m-%d %H:%M")
-            match sizes[i][1].strip():
+                dt.replace(tzinfo=datetime.timezone.utc)
-                case '-':
+                times[i] = time.mktime(dt.timetuple())
-                    sizes[i] = 0
+            for i in range(len(sizes)):
-                case 'K':
+                match sizes[i][1].strip():
-                    sizes[i] = int(float(sizes[i][0].strip()) * 1024)
+                    case '-':
-                case 'M':
+                        sizes[i] = 0
-                    sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024)
+                    case 'K':
-                case 'G':
+                        sizes[i] = int(float(sizes[i][0].strip()) * 1024)
-                    sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024 * 1024)
+                    case 'M':
-                case "0":
+                        sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024)
-                    sizes[i] = 0
+                    case 'G':
-                case "":
+                        sizes[i] = int(float(sizes[i][0].strip()) * 1024 * 1024 * 1024)
-                    sizes[i] = int(float(sizes[i][0].strip()))
+                    case "0":
-                case _:
+                        sizes[i] = 0
-                    raise(ValueError(f"Unexpected symbol while parsing links page: {_}"))
+                    case "":
+                        sizes[i] = int(float(sizes[i][0].strip()))
+                    case _:
+                        raise(ValueError(f"Unexpected symbol while parsing links page: {_}"))
 
             
-        if (len(links) != len(times)) or (len(times) != len(sizes)):
+            if (len(links) != len(times)) or (len(times) != len(sizes)):
-            raise(ValueError("Links parsing error, mismatched numbers of links, times, or sizes!"))
+                raise(ValueError("Links parsing error, mismatched numbers of links, times, or sizes!"))
 
-        results = list(zip(links, times, sizes))
+            results = list(zip(links, times, sizes))
-        if randomize_order:
+            if randomize_order:
-            random.shuffle(results)
+                random.shuffle(results)
-        for _link, _time, _size in results:
+            for _link, _time, _size in results:
-            if _link.endswith("/"):
+                if _link.endswith("/"):
-                if _link.split(r"/")[-2] in ignore_folder_names:
+                    if _link.split(r"/")[-2] in ignore_folder_names:
-                    continue
+                        continue
+                    else:
+                        # print(f"Queueing folder: {urllib.parse.urljoin(url,_link)}")
+                        query_work_queue.put(urllib.parse.urljoin(url,_link))
                 else:
-                    query_work_queue.put(urllib.parse.urljoin(url,_link))
+                    # print(f"Queueing file: {urllib.parse.urljoin(url,_link)}")
-            else:
+                    query_result_queue.put((urllib.parse.urljoin(url,_link), _time, _size))
-                query_result_queue.put((urllib.parse.urljoin(url,_link), _time, _size))
     
 
 # Fetch file from url and store it to path, retrying on failure
@@ -103,11 +107,15 @@ def file_download_worker(download_work_queue, download_result_queue, attempt_cou
             return
         url, path, t, s = job
 
+        # print(f"Next job: {url}, {path}, {t}, {s}")
         attempts = 0
         while attempts < attempt_count:
             try:
+                starttime = time.time()
                 req = urllib.request.Request(url, data=None)
-                image_data = urllib.request.urlopen(req).read()
+                image_data = urllib.request.urlopen(req, timeout=10.0).read()
+                endtime = time.time()
+                tqdm.tqdm.write(f"Downloaded {url} in {endtime-starttime:0.2f} s | {len(image_data)/1024/1024/(endtime-starttime):0.2f} MB/s")
                 if math.isclose(len(image_data), s, rel_tol=0.05):
                     os.makedirs(os.path.split(path)[0], exist_ok=True)
                     open(path, 'wb').write(image_data)
@@ -118,14 +126,15 @@ def file_download_worker(download_work_queue, download_result_queue, attempt_cou
             except Exception as e:
                 if hasattr(e, "code") and e.code == 404: # This is expected if the file has been removed from the site (at least for swpc.noaa.gov)
                     attempts += attempt_count
-                elif attempts == 0:
+                elif (0 < attempts) and (attempts < attempt_count):
-                    print(f"\nA problem occurred on image: {url} | {e}")
+                    # tqdm.tqdm.write(f"\nA problem occurred on file: {url} | {e}")
                     time.sleep(1 + random.random())
                     attempts += 1
-            if (attempt_count > 1) and (attempts == attempt_count):
+                if (attempt_count > 1) and (attempts == attempt_count):
-                print(f'\nAfter {attempt_count} retries, could not fetch: {url}')
+                    tqdm.tqdm.write(f'\nAfter {attempt_count} retries, could not fetch: {url}')
-                download_result_queue.put((False, url, t))
+                    tqdm.tqdm.write(f"Exception: {e}")
-                break
+                    download_result_queue.put((False, url, t))
+                    break
 
 
 if __name__ == "__main__":
@@ -141,8 +150,8 @@ if __name__ == "__main__":
 
     query_work_queue = queue.Queue()
     query_result_queue = queue.Queue()
-    download_work_queue = queue.Queue()
+    download_work_queue = queue.Queue(maxsize=ndownloadworkers)
-    download_result_queue = queue.Queue()
+    download_result_queue = queue.Queue(maxsize=ndownloadworkers)
     workers = []
     for _ in range(nfetchworkers):
         t = Thread(target=find_links_worker, args=(query_work_queue, query_result_queue, 3, randomize_order), daemon=True)
@@ -203,7 +212,7 @@ if __name__ == "__main__":
                                 already_had_image_count += 1
                                 continue
                             else:
-                                print(f'Found a mismatched size on file: {filepath}  Redownloading!')
+                                tqdm.tqdm.write(f'Found a mismatched size on file: {filepath}  Redownloading!')
                     download_work_queue.put((l, filepath, t, s))
                 else:
                     # We have a download record, confirm the file actually exists on disk
@@ -236,23 +245,29 @@ if __name__ == "__main__":
         print(f"Unhandled Exception during run: {e}")
         print("Shutting down")
 
-    with open(file_database_path, 'w') as f:
-        f.write(json.dumps(file_info_cache))
-
     for _ in range(nfetchworkers):
         try:
-            query_work_queue.put(None, timeout=5.0)
+            query_work_queue.put(None)
         except:
             break
 
     time.sleep(1)
     print(f"Downloaded {fetched_image_count} | Already had {already_had_image_count} | Failed {failed_image_count}")
 
+    while len(download_work_queue.queue) > 0:
+        print(f"Waiting for {len(download_work_queue.queue)} downloads in queue...")
+        time.sleep(1)
+
     for _ in range(ndownloadworkers):
         try:
             download_work_queue.put(None, timeout=5.0)
         except:
             break
 
+    with open(file_database_path, 'w') as f:
+        f.write(json.dumps(file_info_cache))
+
+    print("Waiting for workers to shutdown...")
+
     for w in workers:
         w.join(5.0)

requirements.txt

@@ -0,0 +1,8 @@
+astropy
+scikit-image
+tqdm
+numpy
+opencv-python
+palettable
+matplotlib
+pillow