src/streamlit_app.py

import streamlit as st
import pandas as pd
import altair as alt
import json
from pathlib import Path
#local test (put in terminal... if u dont knwo how to do that oh well)
#python -m streamlit run src/streamlit_app.py

root = Path(__file__).resolve().parent.parent
tiles_path = root / "data" / "tiles"

#title and authors and setup
st.set_page_config(page_title= "Mass Surveillance Across America", layout= "wide")
chartwidth= 950
mapheight= 560
chartheight= 420
st.title("Mass Surveillance Across America")
st.subheader("Visualizing Growing Camera Prevalence in the United States")
#fix ur names im lazy its 4am
st.markdown("""Authors: Thomas Southey, Sam, Tergel, Esther Valentin""")

#intro
st.markdown("Across the United States, thousands of cameras are embedded into the infrasturcture of our everyday lives, mounted in our neighborhoods, street poles, parking lots, and highways there are roughly 70,000 cameras within this dataset alone. This project sets out to map and visualize that network, drawing on a public dataset compiled through a GitHub project called FLOCK, which aggregates camera location data from multiple open-source contributors, including OpenStreetMap, the Electronic Frontier Foundation's Atlas of Surveillance, and community-sourced Flock Safety camera locations. The original dataset included international camera records, but for the purposes of this article and its visualizations, we have narrowed our focus to camera records within the United States.  ")
#  Intro& Purpose
# -Explain what the FLOCK dataset is, where it comes from, and why it matters.
# -Intro to the idea of surveillance infrastructure and the ecosystem around it (manufacturers, operators, etc. and how they sell that data to govt)
# -Explain why mapping cameras can changes public perception by making them aware of this invasive surveillance.
# -Basically that seeing a camera here and there seems miniscule, but when u zoom out its fuckin crazy
# -tdlr this project looks at the sheer scale rather than identifying individuals
st.divider()

#load data
#mostly just stole this from what i already sent yall b4 and in my invdiv project
#if u dont like it go ham lol its probably a little wrong or exess, if ur feeling pythonic
@st.cache_data
def loadtiles():
    files= list(tiles_path.rglob("*.json"))
    rows= []
    for file in files:
        if file.name== "index.json":
            continue
        with open(file, "r", encoding= "utf-8") as f:
            tile= json.load(f)
        if "features" not in tile:
            continue
        for feature in tile["features"]:
            props= feature.get("properties", {})
            geom= feature.get("geometry", {})
            if geom.get("type")!= "Point":
                continue
            coords= geom.get("coordinates", [None, None])
            if coords[0] is None or coords[1] is None:
                continue
            rows.append({
                "lon": coords[0],
                "lat": coords[1],
                "operator": props.get("operator", "unknown"),
                "name": props.get("name", "unknown"),
                "manufacturer": props.get("manufacturer", "unknown"),
                "type": props.get("camera:type", props.get("type", props.get("surveillance:type", "unknown"))),
                "direction": props.get("camera:direction", "unknown"),
                "website": props.get("contact:webcam", "unknown")})
    return pd.DataFrame(rows)

df = loadtiles()

#clean data
df["operator"]= df["operator"].fillna("unknown").replace("", "unknown")
df["manufacturer"]= df["manufacturer"].fillna("unknown").replace("", "unknown")
df["name"] = df["name"].fillna("unknown").replace("", "unknown")
df["type"] = df["type"].fillna("unknown").replace("", "unknown")

#lock to america only for the map viz. explained later better.
df= df[(df["lon"] >= -130) & (df["lon"] <= -60) & (df["lat"] >= 20) & (df["lat"] <= 55)]

#ai helped me fix the dupe names it was pmo, but i manually did a few weird ones
#i specifically just printed all unique company names and asked
#it to make a dictionary to overwrite the bazillion unique name
#variations with the most reasonable company name.
#This didn't help as much as i wanted with the unknowns...i would appreciate any help with that. 
companyfix = {
    #mine
    "Q108485435": "Unknown",
    "wikidata=Q108485435": "Unknown",
    "wikidata=q108485435": "Unknown",
    "wikidata=Q135925643": "Unknown",

    #ai helped me with these
    "?": "Unknown",
    "UNKNOWN": "Unknown",
    "unknown": "Unknown",
    "Unknown": "Unknown",
    "unkn": "Unknown",
    "Unkn": "Unknown",
    "Unkwn": "Unknown",

    "FLOCK SAFETY": "Flock Safety",
    "FLOCK Safety": "Flock Safety",
    "Floc Safety": "Flock Safety",
    "Flock": "Flock Safety",
    "flock": "Flock Safety",
    "floc": "Flock Safety",
    "Flock Group Inc.": "Flock Safety",
    "Flock Safetu": "Flock Safety",
    "Flock Saftey": "Flock Safety",
    "Flock Safety Inc": "Flock Safety",
    "Flock Surveillance": "Flock Safety",
    "Flow Safety": "Flock Safety",

    "Motorola": "Motorola Solutions",
    "Motorolla": "Motorola Solutions",
    "Mortorola Solutions": "Motorola Solutions",
    "Motorola/Vigilant": "Motorola Solutions",
    "Motorola?": "Motorola Solutions",

    "AXIS": "Axis Communications",
    "Axis": "Axis Communications",
    "axis": "Axis Communications",
    "Axis Communications AB": "Axis Communications",

    "HIK VISION": "Hikvision",
    "HIK Vision": "Hikvision",
    "HIK vision": "Hikvision",
    "HIKVision": "Hikvision",
    "HK Vision": "Hikvision",
    "HikVision": "Hikvision",
    "hikvision": "Hikvision",
    "Hangzhou Hikvision Digital Technology Co., Ltd.": "Hikvision",
    "NIK VISION": "Hikvision",

    "Hanwha (Wisevision)": "Hanwha",
    "Hanwha Techwin": "Hanwha",
    "Hahwha Techwin": "Hanwha",
    "Wisevision (hanwhavision), série Q ou X": "Hanwha",
    "Wisenet": "Hanwha",
    "Wisenet Hanwha": "Hanwha",

    "Dahua Technology": "Dahua",
    "Dahua Technology Co., Ltd": "Dahua",
    "dahua": "Dahua",
    "Alhua": "Dahua",
    "Alhua technology": "Dahua",
    "alhua": "Dahua",

    "Avigilon": "Avigilon",
    "avigilon": "Avigilon",

    "Ring Inc": "Ring",
    "ring": "Ring",

    "Rekor Systems": "Rekor",
    "Rektor": "Rekor",

    "Reolink Duo 4k 180deg Panorama": "Reolink",
    "realink": "Reolink",
    "reolink": "Reolink",

    "Ubicquia": "Ubicquia",
    "Ubiquiti": "Ubiquiti",
    "ubiquiti": "Ubiquiti",

    "Neology, Inc.": "Neology",
    "Neology, Inc": "Neology",

    "Leonardo US Cyber and Security Solutions, Inc.": "Leonardo",

    "LiveView Technologies": "LVT",
    "LifeView Technologies": "LVT",

    "Verkada Inc.": "Verkada",

    "ZTE Netview": "ZTE NetView",

    "Dekom": "DEKOM",

    "cyber Secure": "Cyber Secure",
    "Cyber Secure?": "Cyber Secure",
    "yber Secure?": "Cyber Secure"
}
df["company"] = df["manufacturer"].replace(companyfix)
st.markdown(f"""This loaded **{len(df):,} cameras** from the FLOCK files.""")

#main viz desc
st.header("Are surveilling cameras a growing epidemic?")
st.markdown("Our central interactive visualization allows you explore the FLOCK dataset across the United States by selecting one or more companies from the drop down menu. Each dot on the map represent a single camera record from the dataset correlating with the selected companies on the dropdown. As cameras are plotted with their geographic coordinates, as you toggle with our visualization, you will begin to see different geographic patterns in how vendors have distributed their infrastructure. Our visualization allows you to zoom in and out of the map to get a closer look at specific cities, regions, and states. We encourage you to explore the map with curiosity and a critical eye while receiving a better depiction of the surveillance state in the United States.  ")
#The map is the primary viz for the project.
# explain how to use the dropdown to compare companies,
# u can pan and zoom into cities to see where cameras cluster.
# explain how i locked it to America so the map stays focused.

#company filter
companyorder = df["company"].value_counts().index.tolist()
companies = companyorder
selectedcompanies = st.multiselect("Select companies", options= companies, default= companies[:5], help= "The top 5 are shown for clarity, but use this to filter to select which companies as you like :)")
if selectedcompanies:
    mapdf = df[df["company"].isin(selectedcompanies)]
else:
    mapdf = df.copy()

#main viz code
mapdf = mapdf.sample(min(len(mapdf), 5000), random_state= 676767) #smaller set for performance bc ik my pc is a fair bit better than others laptops
#i locked the map to amercia only bc we are discussing the US.
mapchart = alt.Chart(mapdf).mark_circle(size= 8, opacity= 0.35).encode(
    x= alt.X("lon:Q", title= "Longitude", scale= alt.Scale(domain= [-130, -60])),
    y= alt.Y("lat:Q", title= "Latitude", scale= alt.Scale(domain= [20, 55])),
    color= alt.Color("company:N",  title= "Company", sort= companyorder,  scale= alt.Scale(domain= companyorder), legend= alt.Legend(columns= 2)),
    tooltip= ["name", "company", "operator", "type", "direction", "lat", "lon"]).properties(width= chartwidth, height= mapheight, title= "Camera Records in the United States").interactive() 
st.altair_chart(mapchart, width= chartwidth)

st.markdown("Not all surveillance cameras are created equally, and not all of the companies behind them operate the same way. The following two visualizations provide a depiction into the top camera operators, followed by the most common types of cameras which are ultimately differentiated through their purpose. In the bar chart below, the two most common companies are labeled as “Unknown” and Flock Safety. The unknown label represents cameras whose manufacture could not be identified, while Flock Safety is a company that markets its camera products primarily to law enforcement agencies, private communities, and homeowners associations. The association between the companies that deploy these cameras with specific purposes reflects a deliberate system shaped by the interests and contracts of the institutions that fund and operate it.  ")
# Distribution over america
# -Look at grouping around cities like our very own chicago or nyc/cali etc, and major intersections. 
# -Look at how suburbs have differnt concentrations than rural areas.
# -Ask user or show user how zooming into these cities should change the viewer's perception
# -Why is the distribution so uneaven? (TALK ABOUT REDLINING AND HOW THESE CAMS EXISTS MOSTLY IN LOWER SOCIOECONIC ZONES AND MINORITY NEIGHBORHOODS, AND HOW THIS IS A PROBLEM)

st.divider()

#context viz 1
st.header("Most Common Companies?")
opcounts = df["company"].value_counts().head(15).reset_index()
opcounts.columns = ["company", "count"]

opchart = alt.Chart(opcounts).mark_bar().encode(
    x= alt.X("count:Q", title= "Number of camera records"), y= alt.Y("company:N", title= "company", sort= "-x"),
    tooltip= ["company", "count"]).properties(width= chartwidth, height= chartheight, title= "Top companys in the Dataset")
st.altair_chart(opchart, width= chartwidth)

#st.markdown("lwokey find a new plot to do this is not good")
#st.caption("fix this caption later or delete it idrc")
st.divider()

#context viz 2
st.header("What types of cameras are we looking at?")
typecounts= df["type"].value_counts().head(12).reset_index()
typecounts.columns = ["type", "count"]

typechart = alt.Chart(typecounts).mark_bar().encode(
    x= alt.X("type:N", title= "Camera type", sort= "-y"), y= alt.Y("count:Q", title= "Number of records"), color= alt.Color("type:N", legend= None),
    tooltip= ["type", "count"]).properties(width= chartwidth, height= chartheight, title= "Most Common Camera Types")

st.altair_chart(typechart,  width= chartwidth)

st.markdown("These camera types reflect a broader trend in public safety technology that acknowledges where the line between observation and intervention is increasingly thin. Understanding what kinds of cameras exist, and in what quantities, is a foundational step in being able to have an informed public conversation about where and how these tools are used. ")
# Company dominance and how we did data cleaning
# -metadata was very inconsistent or missing
# -there were many typo corrections or company names to consolidate.
# -Discuss Flock Safety as the big dawg.
# -Explain why our data cleaning matters for visualization for the public (whatever that means i just am reading the rubric)
# -what are the limitations of data like this (its a crowdsourced and publicly accumulated dataset)
st.caption("Visualization created via the FLOCK camera dataset.")
#come up with better captions my brain was fried at 4am
st.divider()

#extra context
st.header("Why You Should Care")
#once again fix these captions lol
st.markdown("Surveillance infrastructure is not neutral. The choices made about where to place cameras, which companies contract with, and what data to retain are decisions made by institutions and can have real consequences for the communities living within the field of view. Historically, the communities who are most heavily surveilled have also been the most marginalized, this includes low-income neighborhoods, communities of color, and densely populated urban areas (AmericanBarOrg, 2024). Companies such as Flock Safety use cameras such as ALPR’S (Automated License Plate Recognition cameras), don’t simply record, but build a database of movement that can be shared across law enforcement agencies with little public oversight or consent from the people being tracked. As the implementation of this technology steadily increases, its widespread adaption expands into spaces that were considered to be unmonitored and private. These visualizations hope to be a starting point for critical awareness and conversation surrounding surveillance architecture and the communities it most deeply impacts. ")
#  our interpretation and the implications
# -Explain how surveillance networks operate at such a massive scale with little (now growing) public awareness check out eyesonflock on google too.
# -What questions did yall have about data ownership and access to surveillance data
# -Mention public vs private surveillance and how the lines are nonexistant. (private companies sell massive datasets to govt for profit rather than catching bad guys.)
# -The viz works better to prove the point and inform rather than advocate some change or whatever altho u can at EFF if u google that.
# -Explain the importance of data transparency, and soutions to this "dragnet" surveillance (i.e. public awareness, policy change, etc. once agian visit EFF)

#sources
st.header("Sources")
st.markdown("""
- FLOCK camera dataset: https://github.com/Ringmast4r/FLOCK
- Visualizations created by us using Python, Altair, and Streamlit.
- Dataset was loaded from the master repository tiled JSON files to avoid the 100mb limit.
- https://www.americanbar.org/groups/crsj/resources/human-rights/2024-june/mass-surveillance-dangerous-american-communities-reforming-section-702/ """)