Synced repo using 'sync_with_huggingface' Github Action

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/unc_and_mai_combined.xlsx filter=lfs diff=lfs merge=lfs -text

Readme.md

@@ -0,0 +1,65 @@
+# Data for Macroeconomic Attention and Narrative Uncertainty
+
+We provide the daily and monthly macroeconomic attention indices (MAI) and narrative uncertainty scores (UNC) for the New York Times (NYT_) and Wall Street Journal (WSJ_) used in the following papers:
+- Fisher, Martineau, and Sheng (2021) "Macroeconomic Attention and Announcement Risk Premia" [SSRN](https://ssrn.com/abstract=2703978)
+- Martineau, Poulos, Wu, Thompson, Haghighi, Yuan, Hull (2023) "Narrative Monetary Policy Uncertainty" [SSRN](https://ssrn.com/abstract=4573829)
+- Benson, Hull, Nozawa, Poulos, Strela, Wu (2023) "Forecasting Using Text-Based Uncertainty Measures" [SSRN]()
+
+The visualization of the data on daily and monthly frequency can be viewed on [this GitHub page]().
+
+## Data
+The MAI data is available from June 1980 to December 2020, and includes the following series:
+- CREDIT_RATING_MAI: credit rating
+- GDP_MAI: GDP
+- HSNG_MAI: housing market
+- INFL_MAI: inflation
+- MONETARY_MAI: monetary
+- OIL_MAI: oil
+- UNEMP_MAI: unemployment
+- USD_MAI: USD
+
+The UNC data is available from January 1984 to May 2023, we include both uncertainty generated using static Loughran McDonald dictionary (_LM) and uncertainty generated using [BERT model](https://huggingface.co/jeniakim/hedgehog) (_ML). The data includes the following series:
+- NMPU: monetary policy uncertainty
+- INFU: inflation uncertainty
+- INFU_weighted: the daily average of inflation uncertainty is calculated based on the weighted average of counts of the word "inflation" in the article
+- INFU_screened: the daily average of inflation uncertainty is calculated only on articles where the word "inflation" appears once in the title or the first five sentences in the text, or when the word "inflation" appears at least twice in the article
+- GSCI: uncertainty about commodity, referenced as GSCU in our article.
+
+We provide the daily series, daily series detrended by 252 trading day average, and monthly trading day average series.
+
+We also include the following existing measures of uncertainty. If you use those data, please cite the corresponding works accordingly.
+- EPU: policy uncertainty measure proposed by Baker and Bloom. [Official website](http://www.policyuncertainty.com/)
+- MPU-MKT: Uncertainty from short-term rates based on Eurodollar futures options by [Bauer](https://www.michaeldbauer.com/files/mpu.pdf)
+- VIX: Chicago Board Options Exchange Volatility Index, monthly data is the last valid MOVE of each month
+- MOVE: Merrill Lynch Option Volatility Estimate, monthly data is the last valid MOVE of each month
+
+## Citation
+If you use this data, please cite our studies accordingly:
+```
+@article{MAI,
+    author = {Fisher, Adlai and Martineau, Charles and Sheng, Jinfei},
+    title = "{Macroeconomic Attention and Announcement Risk Premia}",
+    journal = {Review of Financial Studies},
+    volume = {35},
+    number = {11},
+    pages = {5057-5093},
+    year = {2022},
+    month = {02},
+    abstract = "{We construct macroeconomic attention indexes (MAI), which are new measures of attention to different macroeconomic risks, including unemployment and monetary policy. Individual MAI tend to increase around related announcements and following changes in related fundamentals. Further, bad news raises attention more than good news. For unemployment and FOMC, attention predicts announcement risk premiums and implied volatility changes with large economic magnitudes. Our findings support theories of endogenous attention and announcement risk premiums, while demonstrating future research directions, including that announcements can raise new concerns. Macroeconomic announcements are important not only for contents and timing but also for attention.Authors have furnished an Internet Appendix, which is available on the Oxford University Press Web site next to the link to the final published paper online.}",
+    issn = {0893-9454},
+    doi = {10.1093/rfs/hhac011},
+    url = {https://doi.org/10.1093/rfs/hhac011},
+    eprint = {https://academic.oup.com/rfs/article-pdf/35/11/5057/46558642/hhac011.pdf},
+}
+
+@article{NMPU,
+  author={Martineau, Charles and Poulos, Zissis and Wu, Yuntao and Thompson, Cameron and Haghighi, Maryam and Yuan, Jun and Hull, John},
+  title={Narrative Monetary Policy Uncertainty},
+  year={2023},
+  month={9},
+  note = {Available at SSRN: \url{https://ssrn.com/abstract=4573829} or \url{http://dx.doi.org/10.2139/ssrn.4573829}}
+}
+```
+
+## Credit
+This project is built on [Gradio](https://github.com/gradio-app/gradio), [Plotly](https://github.com/plotly/plotly.py) and [pandas](https://pandas.pydata.org/)

app.py

@@ -0,0 +1,282 @@
+from data_preproc import *
+import gradio as gr
+import plotly.express as px
+
+BASE_CSS = """
+a:link {
+    color: blue;
+    text-decoration:underline;
+}
+a:hover {
+    color: red;
+    text-decoration:underline;
+}
+hr {
+    display: block;
+    height: 1px;
+    border: 0;
+    border-top: 1px solid #ccc;
+    margin: 1em 0;
+    padding: 0;
+}
+"""
+
+def onDownloadButtonClick():
+    return "data/unc_and_mai_combined.xlsx"
+
+def onCheckboxChange(selected_cols, base_df, title, y_title, smooth, smooth_window):
+    # df = base_df[["date"] + selected_cols]
+    df = base_df[["date"] + selected_cols].copy()
+    corr_df = calculate_correlation_df(df, selected_cols)
+    if smooth:
+        df = smooth_df(df, selected_cols, smooth_window)
+    # melted_df = df.melt(id_vars="date", value_vars=selected_cols, var_name="type", value_name="value")
+    fig = px.line(df, x="date", y=selected_cols, title=title, width=1000, height=500).update_layout(xaxis_title="date", yaxis_title=y_title)
+    fig.update_xaxes(rangeslider_visible=True)
+    for recession in NBER_recession_dates:
+        fig.add_vrect(recession["start"], recession["stop"], fillcolor="gray", opacity=0.2)
+    
+    return gr.Plot(fig), gr.DataFrame(corr_df, label="correlation")
+
+def main():
+     with gr.Blocks(css=BASE_CSS, title="Macroeconomic Attention and Narrative Uncertainty") as block:
+        with gr.Tab("Introduction"):
+            gr.Markdown("""# Macroeconomic Attention and Narrative Uncertainty  
+We provide the daily and monthly macroeconomic attention indices (MAI) and narrative uncertainty scores (UNC) for the New York Times (NYT_) and Wall Street Journal (WSJ_) used in the following papers:  
+- Fisher, Martineau, and Sheng (2021) "Macroeconomic Attention and Announcement Risk Premia" [SSRN](https://ssrn.com/abstract=2703978)  
+- Martineau, Poulos, Wu, Thompson, Haghighi, Yuan, Hull (2023) "Narrative Monetary Policy Uncertainty" [SSRN](https://ssrn.com/abstract=4573829)  
+                        
+## Data
+The MAI data is available from June 1980 to December 2020, and includes the following series:
+- CREDIT_RATING_MAI: credit rating
+- GDP_MAI: GDP
+- HSNG_MAI: housing market
+- INFL_MAI: inflation
+- MONETARY_MAI: monetary
+- OIL_MAI: oil
+- UNEMP_MAI: unemployment
+- USD_MAI: USD
+
+The UNC data is available from January 1984 to May 2023, we include both uncertainty generated using static Loughran McDonald dictionary (_LM) and uncertainty generated using [BERT model](https://huggingface.co/jeniakim/hedgehog) (_ML). The data includes the following series:
+- NMPU: monetary policy uncertainty
+- INFU: inflation uncertainty
+- INFU_weighted: the daily average of inflation uncertainty is calculated based on the weighted average of counts of the word "inflation" in the article
+- INFU_screened: the daily average of inflation uncertainty is calculated only on articles where the word "inflation" appears once in the title or the first five sentences in the text, or when the word "inflation" appears at least twice in the article
+- GSCI: uncertainty about commodity, referenced as GSCU in our article.
+                        
+We also include the following existing measures of uncertainty. If you use those data, please cite the corresponding works accordingly.
+- EPU: policy uncertainty measure proposed by Baker and Bloom. [Official website](http://www.policyuncertainty.com/)
+- MPU-MKT: Uncertainty from short-term rates based on Eurodollar futures options by [Bauer](https://www.michaeldbauer.com/files/mpu.pdf)
+- VIX: Chicago Board Options Exchange Volatility Index, monthly data is the last valid MOVE of each month
+- MOVE: Merrill Lynch Option Volatility Estimate, monthly data is the last valid MOVE of each month
+
+Besides this introduction tab, there another 4 tabs you can navigate to.  
+- **Daily Series**: The raw data. Full daily series from 1980-2023
+- **Daily Detrended Series**: Trading day data only, detrended by 252 day mean  
+- **Monthly Series**: Average of the trading day value for each month. 
+                        
+Inside these tabs, we show the timeseris plots and correlations.""")
+            gr.File(value="data/unc_and_mai_combined.xlsx", type="filepath", label="Download Full Data")
+
+        with gr.Tab("Daily Series"):
+            gr.Markdown("**Note:** The gray boxs are NBER recessions.")
+            with gr.Group():
+                gr.Markdown("## MAI")
+                mai_check_box = gr.CheckboxGroup(mai_cols, label="MAI indices", interactive=True)
+                mai_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                mai_plot = gr.Plot()
+                mai_corr = gr.DataFrame(label="correlation")
+                mai_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[mai_check_box, gr.State(daily_mai), gr.State("Macroeconomic Attention Index"), gr.State("MAI"), mai_smooth_check_box, gr.State(252)],
+                                    outputs=[mai_plot, mai_corr])
+                mai_check_box.change(onCheckboxChange,
+                                    inputs=[mai_check_box, gr.State(daily_mai), gr.State("Macroeconomic Attention Index"), gr.State("MAI"), mai_smooth_check_box, gr.State(252)],
+                                    outputs=[mai_plot, mai_corr])
+            with gr.Group():
+                gr.Markdown("## UNC LM")
+                lm_check_box = gr.CheckboxGroup(unc_lm_cols, label="UNC_LM indices", interactive=True)
+                lm_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                lm_plot = gr.Plot()
+                lm_corr = gr.DataFrame(label="correlation")
+                lm_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[lm_check_box, gr.State(daily_unc_lm), gr.State("Narrative uncertainty LM"), gr.State("UNC"), lm_smooth_check_box, gr.State(252)],
+                                    outputs=[lm_plot, lm_corr])
+                lm_check_box.change(onCheckboxChange,
+                                    inputs=[lm_check_box, gr.State(daily_unc_lm), gr.State("Narrative uncertainty LM"), gr.State("UNC"), lm_smooth_check_box, gr.State(252)],
+                                    outputs=[lm_plot, lm_corr])
+
+            with gr.Group():
+                gr.Markdown("## UNC ML")
+                ml_check_box = gr.CheckboxGroup(unc_ml_cols, label="UNC_ML indices", interactive=True)
+                ml_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                ml_plot = gr.Plot()
+                ml_corr = gr.DataFrame(label="correlation")
+                ml_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[ml_check_box, gr.State(daily_unc_ml), gr.State("Narrative uncertainty ML"), gr.State("UNC"), ml_smooth_check_box, gr.State(252)],
+                                    outputs=[ml_plot, ml_corr])
+                ml_check_box.change(onCheckboxChange,
+                                    inputs=[ml_check_box, gr.State(daily_unc_ml), gr.State("Narrative uncertainty ML"), gr.State("UNC"), ml_smooth_check_box, gr.State(252)],
+                                    outputs=[ml_plot, ml_corr])
+                
+            with gr.Group():
+                gr.Markdown("## Existing Measures")
+                other_check_box = gr.CheckboxGroup(other_cols, label="Existing indices", interactive=True)
+                other_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                other_plot = gr.Plot()
+                other_corr = gr.DataFrame(label="correlation")
+                other_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[other_check_box, gr.State(daily_other), gr.State("Existing indices"), gr.State("idx"), other_smooth_check_box, gr.State(252)],
+                                    outputs=[other_plot, other_corr])
+                other_check_box.change(onCheckboxChange,
+                                    inputs=[other_check_box, gr.State(daily_other), gr.State("Existing indices"), gr.State("idx"), other_smooth_check_box, gr.State(252)],
+                                    outputs=[other_plot, other_corr])
+            with gr.Group():
+                gr.Markdown("""## All measures
+All the data are normalized to [0,1] by $\\frac{x-x_{\\text{min}}}{x_{\\text{max}-x_{\\text{min}}}}$""", 
+                latex_delimiters=[{ "left": "$$", "right": "$$", "display": True }, { "left": "$", "right": "$", "display": False }])
+                all_check_box = gr.CheckboxGroup(all_cols, label="all measures", interactive=True)
+                all_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                all_plot = gr.Plot()
+                all_corr = gr.DataFrame()
+                all_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[all_check_box, gr.State(daily_df_all), gr.State("All indices"), gr.State("idx"), all_smooth_check_box, gr.State(252)],
+                                    outputs=[all_plot, all_corr])
+                all_check_box.change(onCheckboxChange,
+                                    inputs=[all_check_box, gr.State(daily_df_all), gr.State("All indices"), gr.State("idx"), all_smooth_check_box, gr.State(252)],
+                                    outputs=[all_plot, all_corr])
+                
+        with gr.Tab("Daily Detrended Series"):
+            gr.Markdown("**Note:** The gray boxs are NBER recessions.")
+            with gr.Group():
+                gr.Markdown("## MAI")
+                mai_check_box = gr.CheckboxGroup(mai_cols, label="MAI indices", interactive=True)
+                mai_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                mai_plot = gr.Plot()
+                mai_corr = gr.Dataframe(label="correlation")
+                mai_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[mai_check_box, gr.State(daily_detrend_mai), gr.State("Macroeconomic Attention Index"), gr.State("MAI"), mai_smooth_check_box, gr.State(252)],
+                                    outputs=[mai_plot, mai_corr])
+                mai_check_box.change(onCheckboxChange,
+                                    inputs=[mai_check_box, gr.State(daily_detrend_mai), gr.State("Macroeconomic Attention Index"), gr.State("MAI"), mai_smooth_check_box, gr.State(252)],
+                                    outputs=[mai_plot, mai_corr])
+            with gr.Group():
+                gr.Markdown("## UNC LM")
+                lm_check_box = gr.CheckboxGroup(unc_lm_cols, label="UNC_LM indices", interactive=True)
+                lm_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                lm_plot = gr.Plot()
+                lm_corr = gr.DataFrame(label="correlation")
+                lm_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[lm_check_box, gr.State(daily_detrend_unc_lm), gr.State("Narrative uncertainty LM"), gr.State("UNC"), lm_smooth_check_box, gr.State(252)],
+                                    outputs=[lm_plot, lm_corr])
+                lm_check_box.change(onCheckboxChange,
+                                    inputs=[lm_check_box, gr.State(daily_detrend_unc_lm), gr.State("Narrative uncertainty LM"), gr.State("UNC"), lm_smooth_check_box, gr.State(252)],
+                                    outputs=[lm_plot, lm_corr])
+            with gr.Group():
+                gr.Markdown("## UNC ML")
+                ml_check_box = gr.CheckboxGroup(unc_ml_cols, label="UNC_ML indices", interactive=True)
+                ml_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                ml_plot = gr.Plot()
+                ml_corr = gr.DataFrame(label="correlation")
+                ml_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[ml_check_box, gr.State(daily_detrend_unc_ml), gr.State("Narrative uncertainty ML"), gr.State("UNC"), ml_smooth_check_box, gr.State(252)],
+                                    outputs=[ml_plot, ml_corr])
+                ml_check_box.change(onCheckboxChange,
+                                    inputs=[ml_check_box, gr.State(daily_detrend_unc_ml), gr.State("Narrative uncertainty ML"), gr.State("UNC"), ml_smooth_check_box, gr.State(252)],
+                                    outputs=[ml_plot, ml_corr])
+            with gr.Group():
+                gr.Markdown("## Existing Measures")
+                other_check_box = gr.CheckboxGroup(other_cols, label="Existing indices", interactive=True)
+                other_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                other_plot = gr.Plot()
+                other_corr = gr.Plot()
+                other_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[other_check_box, gr.State(daily_detrend_other), gr.State("Existing indices"), gr.State("idx"), other_smooth_check_box, gr.State(252)],
+                                    outputs=[other_plot, other_corr])
+                other_check_box.change(onCheckboxChange,
+                                    inputs=[other_check_box, gr.State(daily_detrend_other), gr.State("Existing indices"), gr.State("idx"), other_smooth_check_box, gr.State(252)],
+                                    outputs=[other_plot, other_corr])
+            with gr.Group():
+                gr.Markdown("""## All measures
+All the data are normalized to [0,1] by $\\frac{x-x_{\\text{min}}}{x_{\\text{max}-x_{\\text{min}}}}$""", 
+                latex_delimiters=[{ "left": "$$", "right": "$$", "display": True }, { "left": "$", "right": "$", "display": False }])
+                all_check_box = gr.CheckboxGroup(all_cols, label="all measures", interactive=True)
+                all_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                all_plot = gr.Plot()
+                all_corr = gr.DataFrame()
+                all_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[all_check_box, gr.State(daily_detrend_df_all), gr.State("All indices"), gr.State("idx"), all_smooth_check_box, gr.State(252)],
+                                    outputs=[all_plot, all_corr])
+                all_check_box.change(onCheckboxChange,
+                                    inputs=[all_check_box, gr.State(daily_detrend_df_all), gr.State("All indices"), gr.State("idx"), all_smooth_check_box, gr.State(252)],
+                                    outputs=[all_plot, all_corr])
+
+        with gr.Tab("Monthly Series"):
+            gr.Markdown("**Note:** The gray boxs are NBER recessions.")
+            with gr.Group():
+                gr.Markdown("## MAI")
+                mai_check_box = gr.CheckboxGroup(mai_cols, label="MAI indices", interactive=True)
+                mai_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                mai_plot = gr.Plot()
+                mai_corr = gr.DataFrame(label="correlation")
+                mai_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[mai_check_box, gr.State(monthly_mai), gr.State("Macroeconomic Attention Index"), gr.State("MAI"), mai_smooth_check_box, gr.State(12)],
+                                    outputs=[mai_plot, mai_corr])
+                mai_check_box.change(onCheckboxChange,
+                                    inputs=[mai_check_box, gr.State(monthly_mai), gr.State("Macroeconomic Attention Index"), gr.State("MAI"), mai_smooth_check_box, gr.State(12)],
+                                    outputs=[mai_plot, mai_corr])
+            with gr.Group():
+                gr.Markdown("## UNC LM")
+                lm_check_box = gr.CheckboxGroup(unc_lm_cols, label="UNC_LM indices", interactive=True)
+                lm_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                lm_plot = gr.Plot()
+                lm_corr = gr.DataFrame(label="correlation")
+                lm_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[lm_check_box, gr.State(monthly_unc_lm), gr.State("Narrative uncertainty LM"), gr.State("UNC"), lm_smooth_check_box, gr.State(12)],
+                                    outputs=[lm_plot, lm_corr])
+                lm_check_box.change(onCheckboxChange,
+                                    inputs=[lm_check_box, gr.State(monthly_unc_lm), gr.State("Narrative uncertainty LM"), gr.State("UNC"), lm_smooth_check_box, gr.State(12)],
+                                    outputs=[lm_plot, lm_corr])
+            with gr.Group():
+                gr.Markdown("## UNC ML")
+                ml_check_box = gr.CheckboxGroup(unc_ml_cols, label="UNC_ML indices", interactive=True)
+                ml_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                ml_plot = gr.Plot()
+                ml_corr = gr.DataFrame(label="correlation")
+                ml_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[ml_check_box, gr.State(monthly_unc_ml), gr.State("Narrative uncertainty ML"), gr.State("UNC"), ml_smooth_check_box, gr.State(12)],
+                                    outputs=[ml_plot, ml_corr])
+                ml_check_box.change(onCheckboxChange,
+                                    inputs=[ml_check_box, gr.State(monthly_unc_ml), gr.State("Narrative uncertainty ML"), gr.State("UNC"), ml_smooth_check_box, gr.State(12)],
+                                    outputs=[ml_plot, ml_corr])
+            with gr.Group():
+                gr.Markdown("## Existing Measures")
+                other_check_box = gr.CheckboxGroup(other_cols, label="Existing indices", interactive=True)
+                other_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                other_plot = gr.Plot()
+                other_corr = gr.DataFrame(label="correlation")
+                other_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[other_check_box, gr.State(monthly_other), gr.State("Existing indices"), gr.State("idx"), other_smooth_check_box, gr.State(12)],
+                                    outputs=[other_plot, other_corr])
+                other_check_box.change(onCheckboxChange,
+                                    inputs=[other_check_box, gr.State(monthly_other), gr.State("Existing indices"), gr.State("idx"), other_smooth_check_box, gr.State(12)],
+                                    outputs=[other_plot, other_corr])
+            with gr.Group():
+                gr.Markdown("""## All measures
+All the data are normalized to [0,1] by $\\frac{x-x_{\\text{min}}}{x_{\\text{max}-x_{\\text{min}}}}$""", 
+                latex_delimiters=[{ "left": "$$", "right": "$$", "display": True }, { "left": "$", "right": "$", "display": False }])
+                all_check_box = gr.CheckboxGroup(all_cols, label="all measures", interactive=True)
+                all_smooth_check_box = gr.Checkbox(False, label="Smooth")
+                all_plot = gr.Plot()
+                all_corr = gr.DataFrame()
+                all_smooth_check_box.change(onCheckboxChange,
+                                    inputs=[all_check_box, gr.State(monthly_df_all), gr.State("All indices"), gr.State("idx"), all_smooth_check_box, gr.State(12)],
+                                    outputs=[all_plot, all_corr])
+                all_check_box.change(onCheckboxChange,
+                                    inputs=[all_check_box, gr.State(monthly_df_all), gr.State("All indices"), gr.State("idx"), all_smooth_check_box, gr.State(12)],
+                                    outputs=[all_plot, all_corr])
+
+        block.launch(debug=True, server_port=8005)
+
+
+if __name__ == "__main__":
+    main()

data/unc_and_mai_combined.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23ac1ecc2b81a68b8a3c492cabae81df56f5176355524338d01e171edf1a2072
+size 11202969

data_preproc.py

@@ -0,0 +1,97 @@
+import pandas as pd
+
+'''
+Load the data from excel and categorize into MAI/UNC_LM/UNC_ML/existing for 
+daily, daily_detrended and monthly series
+'''
+
+daily_df = pd.read_excel("data/unc_and_mai_combined.xlsx", sheet_name="daily")
+daily_detrend_df = pd.read_excel("data/unc_and_mai_combined.xlsx", sheet_name="daily_detrended")
+monthly_df = pd.read_excel("data/unc_and_mai_combined.xlsx", sheet_name="monthly")
+
+mai_cols = ["WSJ_CREDIT_RATING_MAI", "NYT_CREDIT_RATING_MAI",
+            "WSJ_GDP_MAI", "NYT_GDP_MAI", 
+            "WSJ_HSNG_MAI", "NYT_HSNG_MAI",
+            "WSJ_INFL_MAI", "NYT_INFL_MAI",
+            "WSJ_MONETARY_MAI", "NYT_MONETARY_MAI",
+            "WSJ_OIL_MAI", "NYT_OIL_MAI",
+            "WSJ_UNEMP_MAI", "NYT_UNEMP_MAI",
+            "WSJ_USD_MAI", "NYT_USD_MAI"]
+
+unc_lm_cols = ["WSJ_NMPU_LM", "NYT_NMPU_LM", 
+               "WSJ_INFU_LM", "NYT_INFU_LM",
+            #    "WSJ_INFU_LM_weighted", "NYT_INFU_LM_weighted",
+               "WSJ_INFU_LM_screened", "NYT_INFU_LM_screened",
+               "WSJ_GSCI_LM", "NYT_GSCI_LM"]
+
+unc_ml_cols = ["WSJ_NMPU_ML", "NYT_NMPU_ML", 
+               "WSJ_INFU_ML", "NYT_INFU_ML",
+            #    "WSJ_INFU_ML_weighted", "NYT_INFU_ML_weighted",
+               "WSJ_INFU_ML_screened", "NYT_INFU_ML_screened",
+               "WSJ_GSCI_ML", "NYT_GSCI_ML"]
+
+other_cols = ["EPU", "VIX", "MPU_MKT", "MOVE"]
+
+all_cols = mai_cols + unc_lm_cols + unc_ml_cols + other_cols
+
+daily_df_all = daily_df[["date"] + all_cols].copy()
+daily_detrend_df_all = daily_detrend_df[["date"] + all_cols].copy()
+monthly_df_all = monthly_df[["date"] + all_cols].copy()
+
+daily_df_all[all_cols] = (daily_df_all[all_cols] - daily_df_all[all_cols].min()) / (daily_df_all[all_cols].max() - daily_df_all[all_cols].min())
+daily_detrend_df_all[all_cols] = (daily_detrend_df_all[all_cols] - daily_detrend_df_all[all_cols].min()) / (daily_detrend_df_all[all_cols].max() - daily_detrend_df_all[all_cols].min())
+monthly_df_all[all_cols] = (monthly_df_all[all_cols] - monthly_df_all[all_cols].min()) / (monthly_df_all[all_cols].max() - monthly_df_all[all_cols].min())
+
+################ MAI
+daily_mai = daily_df[["date"] + mai_cols]
+daily_detrend_mai = daily_detrend_df[["date"] + mai_cols]
+monthly_mai = monthly_df[["date"] + mai_cols]
+
+################ UNC_LM
+daily_unc_lm = daily_df[["date"] + unc_lm_cols]
+daily_detrend_unc_lm = daily_detrend_df[["date"] + unc_lm_cols]
+monthly_unc_lm = monthly_df[["date"] + unc_lm_cols]
+
+################ UNC_ML
+daily_unc_ml = daily_df[["date"] + unc_ml_cols]
+daily_detrend_unc_ml = daily_detrend_df[["date"] + unc_ml_cols]
+monthly_unc_ml = monthly_df[["date"] + unc_ml_cols]
+
+################ OTHER
+daily_other = daily_df[["date"] + other_cols]
+daily_detrend_other = daily_detrend_df[["date"] + other_cols]
+monthly_other = monthly_df[["date"] + other_cols]
+
+#source: NBER based Recession Indicators | https://fred.stlouisfed.org/series/USREC
+NBER_recession_dates = [{'start': '2020-03-01','stop': '2020-04-01'},
+                        {'start': '2008-01-01','stop': '2009-06-01'},
+                        {'start': '2001-04-01','stop': '2001-11-01'},
+                        {'start': '1990-08-01','stop': '1991-03-01'},
+                        {'start': '1981-08-01','stop': '1981-11-01'},
+                        {'start': '1980-02-01','stop': '1981-07-01'},
+                        ]
+
+def smooth_df(df: pd.DataFrame, cols: list, smooth_window: int=12):
+    '''
+        Smooth all the timeseries in the plot for a better visual
+        
+        Inputs:
+            df: dataframe to smooth
+            cols: columns to apply the smooth
+            smooth_window: suggest 12 for monthly series, and 252 for daily series
+        Returns:
+            a copy of df with cols smoothed using rolling smooth_window mean
+    '''
+    df_ = df.copy()
+    df_[cols] = df_[cols].rolling(smooth_window, min_periods=1).mean(numeric_only=True)
+    return df_
+
+def calculate_correlation_df(df: pd.DataFrame, cols: list):
+    '''
+        Calculates the correlation between the provided columns in the dataframe
+
+        Returns:
+            a pandas df of correlation
+    '''
+    df_ = df[cols].copy()
+    return df_.corr().round(3).astype(str).reset_index()

requirements.txt

@@ -0,0 +1,7 @@
+numpy
+pandas
+openpyxl
+matplotlib
+plotly
+gradio
+nbformat