defmodule PlausibleWeb.Live.CSVImport do
@moduledoc """
LiveView allowing uploading CSVs for imported tables to S3 or local storage
"""
use PlausibleWeb, :live_view
alias Plausible.Imported.CSVImporter
alias Plausible.Imported
# :not_mounted_at_router ensures we have already done auth checks in the controller
# if this liveview becomes available from the router, please make sure
# to check that site_role is allowed to make site imports
@impl true
def mount(:not_mounted_at_router, session, socket) do
%{"site_id" => site_id, "storage" => storage} = session
upload_opts = [
accept: [".csv", "text/csv"],
auto_upload: true,
max_entries: length(Imported.tables()),
# 1GB
max_file_size: 1_000_000_000,
progress: &handle_progress/3
]
upload_opts =
case storage do
"s3" -> [{:external, &presign_upload/2} | upload_opts]
"local" -> upload_opts
end
upload_consumer =
case storage do
"s3" ->
fn meta, entry ->
{:ok, %{"s3_url" => meta.s3_url, "filename" => entry.client_name}}
end
"local" ->
local_dir = CSVImporter.local_dir(site_id)
File.mkdir_p!(local_dir)
fn meta, entry ->
local_path = Path.join(local_dir, Path.basename(meta.path))
Plausible.File.mv!(meta.path, local_path)
{:ok, %{"local_path" => local_path, "filename" => entry.client_name}}
end
end
%{assigns: %{site: site}} =
socket = assign_new(socket, :site, fn -> Plausible.Repo.get!(Plausible.Site, site_id) end)
# we'll listen for new completed imports to know
# when to reload the occupied ranges
if connected?(socket), do: Imported.listen()
occupied_ranges = Imported.get_occupied_date_ranges(site)
native_stats_start_date = Plausible.Sites.native_stats_start_date(site)
socket =
socket
|> assign(
site_id: site_id,
storage: storage,
upload_consumer: upload_consumer,
occupied_ranges: occupied_ranges,
native_stats_start_date: native_stats_start_date
)
|> allow_upload(:import, upload_opts)
|> process_imported_tables()
{:ok, socket}
end
@impl true
def render(assigns) do
~H"""
"""
end
defp csv_picker(assigns) do
~H"""
"""
end
defp confirm_button(assigns) do
~H"""
<.button type="submit" disabled={not @can_confirm?} class="w-full">
<%= if @date_range do %>
Confirm import <.dates range={@date_range} />
<% else %>
Confirm import
<% end %>
"""
end
defp maybe_date_range_warning(assigns) do
~H"""
<%= if @clamped do %>
<.notice :if={@clamped != @original} title="Dates Adjusted" theme={:yellow} class="mt-4">
The dates <.dates range={@original} />
overlap with previous imports, so we'll use the next best period, <.dates range={@clamped} />
<% else %>
<.notice title="Dates Conflict" theme={:red} class="mt-4">
The dates <.dates range={@original} />
overlap with existing site data and cannot be used for a new import.
<% end %>
"""
end
defp dates(assigns) do
~H"""
{@range.first}
to {@range.last}
"""
end
defp imported_table(assigns) do
status =
cond do
assigns.upload && assigns.upload.progress == 100 -> :success
assigns.upload && assigns.upload.progress > 0 -> :in_progress
not Enum.empty?(assigns.errors) -> :error
true -> :empty
end
assigns = assign(assigns, status: status)
~H"""
<.spinner :if={@status == :in_progress} class="w-4 h-4" />
<%= if @upload do %>
{@upload.client_name}
<% else %>
{@table}_YYYYMMDD_YYYYMMDD.csv
<% end %>
{error_to_string(error)}
"""
end
@impl true
def handle_event("validate-upload-form", _params, socket) do
{:noreply, process_imported_tables(socket)}
end
def handle_event("submit-upload-form", _params, socket) do
%{
storage: storage,
site: site,
current_user: current_user,
clamped_date_range: clamped_date_range,
upload_consumer: upload_consumer
} =
socket.assigns
uploads = consume_uploaded_entries(socket, :import, upload_consumer)
{:ok, _job} =
CSVImporter.new_import(site, current_user,
start_date: clamped_date_range.first,
end_date: clamped_date_range.last,
uploads: uploads,
storage: storage
)
redirect_to =
Routes.site_path(socket, :settings_imports_exports, site.domain)
{:noreply, redirect(socket, to: redirect_to)}
end
@impl true
def handle_info({:notification, :analytics_imports_jobs, details}, socket) do
site = socket.assigns.site
socket =
if details["site_id"] == site.id and details["event"] == "complete" do
occupied_ranges = Imported.get_occupied_date_ranges(site)
socket |> assign(occupied_ranges: occupied_ranges) |> process_imported_tables()
else
socket
end
{:noreply, socket}
end
defp error_to_string(:too_large), do: "is too large (max size is 1 gigabyte)"
defp error_to_string(:too_many_files), do: "too many files"
defp error_to_string(:not_accepted), do: "unacceptable file types"
defp error_to_string(:external_client_failure), do: "browser upload failed"
defp presign_upload(entry, socket) do
%{s3_url: s3_url, presigned_url: upload_url} =
Plausible.S3.import_presign_upload(socket.assigns.site_id, random_suffix(entry.client_name))
{:ok, %{uploader: "S3", s3_url: s3_url, url: upload_url}, socket}
end
defp random_suffix(filename) do
# based on Plug.Upload.path/2
# https://github.com/elixir-plug/plug/blob/eabf0b9d43060c10663a9105cb1baf984d272a6c/lib/plug/upload.ex#L154-L159
sec = Integer.to_string(:os.system_time(:second))
rand = Integer.to_string(:rand.uniform(999_999_999_999))
scheduler_id = Integer.to_string(:erlang.system_info(:scheduler_id))
filename <> "-" <> sec <> "-" <> rand <> "-" <> scheduler_id
end
defp handle_progress(:import, entry, socket) do
if entry.done? do
{:noreply, process_imported_tables(socket)}
else
{:noreply, socket}
end
end
defp process_imported_tables(socket) do
tables = Imported.tables()
{completed, in_progress} = uploaded_entries(socket, :import)
{valid_uploads, invalid_uploads} =
Enum.split_with(completed ++ in_progress, &CSVImporter.valid_filename?(&1.client_name))
imported_tables_all_uploads =
Enum.map(tables, fn table ->
uploads =
Enum.filter(valid_uploads, fn upload ->
CSVImporter.extract_table(upload.client_name) == table
end)
{upload, replaced_uploads} = List.pop_at(uploads, -1)
{table, upload, replaced_uploads}
end)
imported_tables =
Enum.map(imported_tables_all_uploads, fn {table, upload, _replaced_uploads} ->
{table, upload}
end)
replaced_uploads =
Enum.flat_map(imported_tables_all_uploads, fn {_table, _upload, replaced_uploads} ->
replaced_uploads
end)
original_date_range = CSVImporter.date_range(Enum.map(valid_uploads, & &1.client_name))
clamped_date_range =
if original_date_range do
%Date.Range{first: start_date, last: end_date} = original_date_range
%{
site: site,
occupied_ranges: occupied_ranges,
native_stats_start_date: native_stats_start_date
} = socket.assigns
cutoff_date = native_stats_start_date || Plausible.Times.today(site.timezone)
case Imported.clamp_dates(occupied_ranges, cutoff_date, start_date, end_date) do
{:ok, start_date, end_date} -> Date.range(start_date, end_date)
{:error, :no_time_window} -> nil
end
end
all_uploaded? = completed != [] and in_progress == []
can_confirm? = all_uploaded? and not is_nil(clamped_date_range)
socket
|> cancel_uploads(invalid_uploads)
|> cancel_uploads(replaced_uploads)
|> assign(
imported_tables: imported_tables,
can_confirm?: can_confirm?,
original_date_range: original_date_range,
clamped_date_range: clamped_date_range
)
end
defp cancel_uploads(socket, uploads) do
Enum.reduce(uploads, socket, fn upload, socket ->
cancel_upload(socket, :import, upload.ref)
end)
end
end