cftc_data/data_download.py

import cot_reports as cot
import streamlit as st
import numpy as np
import plotly.graph_objects as go
import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random

import seaborn as sns
import matplotlib.pyplot as plt
import base64
import requests
import json
def getData():
    df = pd.DataFrame()
    begin_year = 2017
    end_year = 2023
    for i in range(begin_year, end_year + 1):
        single_year = pd.DataFrame(cot.cot_year(i, cot_report_type='legacy_futopt')) 
        df = df.append(single_year, ignore_index=True)
    df1 = df[['Market and Exchange Names','As of Date in Form YYYY-MM-DD','Noncommercial Positions-Long (All)','Noncommercial Positions-Short (All)','Change in Noncommercial-Long (All)','Change in Noncommercial-Short (All)','Open Interest (All)','Change in Open Interest (All)','% of Open Interest (OI) (All)','% of OI-Noncommercial-Long (All)','% of OI-Noncommercial-Short (All)']]
    df1.rename(columns={'Market and Exchange Names': 'Asset', 'As of Date in Form YYYY-MM-DD': 'Date', 'Noncommercial Positions-Long (All)': 'Long', 'Noncommercial Positions-Short (All)': 'Short', 'Change in Noncommercial-Long (All)': 'Change Long', 'Change in Noncommercial-Short (All)': 'Change Short', 'Open Interest (All)' : "OI ALL",'Change in Open Interest (All)' : "OI ALL change",'% of Open Interest (OI) (All)': "%OI ALL",'% of OI-Noncommercial-Long (All)': "%OI ALL Long",'% of OI-Noncommercial-Short (All)': "%OI ALL short"}, inplace=True)
    df1["% Long"] = (df1['Long'] / (df1['Long'] + df1['Short'])) * 100
    df1["% Short"] = (df1['Short'] / (df1['Long'] + df1['Short'])) * 100
    df1["Net Positions"] = df1['Long'] - df1['Short']
    df1["%OI"] = df1["%OI ALL Long"] - df1["%OI ALL short"]

    return df1

df = getData()
df.to_csv("cftc_data.csv")