File size: 5,325 Bytes
8da2481
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
defmodule Plausible.Session.Transfer do
  @moduledoc """
  Cross-deployment transfer for `:sessions` cache.

  It works by establishing a client-server architecture where:
  - The "replica" one-time task retrieves `:sessions` data from other OS processes via Unix domain sockets
  - The "primary" server process responds to requests for `:sessions` data via Unix domain sockets
  - The "alive" process waits on shutdown for at least one replica, for 15 seconds
  """

  @behaviour Supervisor

  require Logger
  alias Plausible.Session.Transfer.{TinySock, Alive}
  alias Plausible.{Cache, ClickhouseSessionV2, Session}

  @cmd_list_cache_names :list
  @cmd_dump_cache :get
  @cmd_takeover_done :done

  def telemetry_event, do: [:plausible, :sessions, :takeover]

  @doc """
  Starts the `:sessions` transfer supervisor.

  Options:
  - `:name` - the name of the supervisor (default: `Plausible.Session.Transfer`)
  - `:base_path` - the base path for the Unix domain sockets (required)
  """
  def start_link(opts) do
    name = Keyword.get(opts, :name, __MODULE__)
    base_path = Keyword.fetch!(opts, :base_path)
    Supervisor.start_link(__MODULE__, base_path, name: name)
  end

  @impl true
  def init(nil) do
    Logger.notice(
      "Session transfer: ignoring, no socket base path configured (make sure ENABLE_SESSION_TRANSFER/PERSISTENT_CACHE_DIR are set)"
    )

    :ignore
  end

  def init(base_path) do
    File.mkdir_p!(base_path)

    replica =
      Supervisor.child_spec(
        {Task, fn -> init_takeover(base_path) end},
        id: :transfer_replica
      )

    given_counter = :counters.new(1, [])
    parent = self()

    primary =
      {TinySock,
       base_path: base_path,
       handler: fn message -> handle_replica(message, parent, given_counter) end}

    alive =
      Supervisor.child_spec(
        {Alive,
         _until = fn ->
           result = :counters.get(given_counter, 1) > 0

           Logger.notice(
             "Session transfer delayed shut down. Checking if session takeover happened?: #{result}"
           )

           result
         end},
        shutdown: :timer.seconds(15)
      )

    Logger.notice("Session transfer init: #{base_path}")
    Supervisor.init([replica, primary, alive], strategy: :one_for_one)
  end

  @doc """
  Returns `true` if the transfer has been attempted (successfully or not).
  Returns `false` if the transfer is still in progress.
  """
  def attempted?(transfer_sup \\ __MODULE__) do
    result = not replica_alive?(transfer_sup)
    Logger.notice("Session transfer attempted?: #{result}")
    result
  end

  @doc """
  Returns the child specification for the `:sessions` transfer supervisor.
  See `start_link/1` for options.
  """
  def child_spec(opts) do
    %{
      id: __MODULE__,
      start: {__MODULE__, :start_link, [opts]},
      type: :supervisor,
      restart: :temporary
    }
  end

  defp handle_replica(request, parent, given_counter) do
    Logger.notice(
      "Session transfer message received at #{node()}: #{inspect(request, limit: 10)}"
    )

    case request do
      {@cmd_list_cache_names, session_version} ->
        if session_version == session_version() and attempted?(parent) do
          Cache.Adapter.get_names(:sessions)
        else
          []
        end

      {@cmd_dump_cache, cache} ->
        Cache.Adapter.cache2list(cache)

      @cmd_takeover_done ->
        :counters.add(given_counter, 1, 1)
    end
  end

  defp init_takeover(base_path) do
    started = System.monotonic_time()

    base_path
    |> TinySock.list!()
    |> Enum.sort_by(&file_stat_ctime/1, :asc)
    |> Enum.each(&request_takeover/1)

    :telemetry.execute(telemetry_event(), %{duration: System.monotonic_time() - started})
  end

  defp request_takeover(sock) do
    Logger.notice("Session transfer: requesting takeover at #{node()}")

    with {:ok, names} <- TinySock.call(sock, {@cmd_list_cache_names, session_version()}) do
      tasks = Enum.map(names, fn name -> Task.async(fn -> takeover_cache(sock, name) end) end)
      Task.await_many(tasks, :timer.seconds(10))
    end
  after
    Logger.notice("Session transfer: marking takeover as done at #{node()}")
    TinySock.call(sock, @cmd_takeover_done)
  end

  defp takeover_cache(sock, cache) do
    Logger.notice("Session transfer: requesting cache #{cache} dump at #{node()}")

    with {:ok, records} <- TinySock.call(sock, {@cmd_dump_cache, cache}) do
      Enum.each(records, fn record ->
        {key, %ClickhouseSessionV2{} = session} = record
        Cache.Adapter.put(:sessions, key, session)
      end)

      Logger.notice("Session transfer: restored cache #{cache} at #{node()}")
    end
  end

  defp file_stat_ctime(path) do
    case File.stat(path) do
      {:ok, stat} -> stat.ctime
      {:error, _} -> nil
    end
  end

  defp session_version do
    [
      ClickhouseSessionV2.module_info(:md5),
      Cache.Adapter.module_info(:md5),
      Session.CacheStore.module_info(:md5),
      Session.Transfer.module_info(:md5)
    ]
  end

  defp replica_alive?(transfer_sup) do
    children = Supervisor.which_children(transfer_sup)

    replica =
      Enum.find_value(children, fn {id, pid, _, _} -> id == :transfer_replica && pid end)

    is_pid(replica) and Process.alive?(replica)
  catch
    :exit, {:noproc, _} -> false
  end
end