feat: save data, make basic plots

convert.py

@@ -0,0 +1,60 @@
+import os
+import yaml
+
+import awkward as ak
+import pandas as pd
+import numpy as np
+import tqdm
+
+scorecard_dir = "data/scorecard"
+scorecard_dir = os.path.join(scorecard_dir, os.listdir(scorecard_dir)[0])
+
+print("Loading data.yaml")
+with open(os.path.join(scorecard_dir, 'data.yaml'), 'r') as file:
+    data = yaml.safe_load(file)
+
+print("Loading CSVs to dataframes")
+files = [f'MERGED{i}_{(i + 1) % 100:02}_PP.csv' for i in tqdm.trange(1996, 2024)]
+dataframes = [pd.read_csv(os.path.join(scorecard_dir, file)) for file in tqdm.tqdm(files)]
+
+print("Appending extra rows where needed")
+unit_ids = np.unique(np.hstack([frame.UNITID.to_numpy() for frame in tqdm.tqdm(dataframes)]))
+for i, frame in tqdm.tqdm(enumerate(dataframes)):
+    new_rows = pd.DataFrame({"UNITID": unit_ids[~np.isin(unit_ids, frame.UNITID)]})
+    dataframes[i] = pd.concat([frame, new_rows]).sort_values(by=["UNITID", "OPEID"])
+
+print("Converting to Results Array")
+result = {}
+for key, sec in tqdm.tqdm(data['dictionary'].items()):
+    if 'calculate' in sec:
+        continue
+        
+    data_key = sec['source']
+    if data_key not in dataframes[0]:
+        continue
+
+    parts = key.split('.')
+    section = result
+    for i in range(len(parts) - 1):
+        part = parts[i]
+        if part not in section:
+            section[part] = {}
+        section = section[part]
+
+    obj = np.vstack([frame[data_key] for frame in dataframes]).T
+    for frame in dataframes:
+        del frame[data_key] # Memory cleanup
+    
+    if obj.dtype == object:
+        obj = obj.astype(str)
+    section[parts[-1]] = obj
+
+print("Cleanup: Deleting Dataframes from Memory")
+del dataframes # Memory cleanup
+
+print("Converting to Awkward Array")
+a = ak.Array(result)
+del result # Memory cleanup
+
+print("Writing to Disk")
+ak.to_parquet(a, os.path.join(scorecard_dir, "merged.parquet"))

requirements.txt

@@ -0,0 +1,5 @@
+awkward
+pandas
+numpy
+
+tqdm