diff --git a/lib/deployex/application.ex b/lib/deployex/application.ex index 088e95c..cff595c 100644 --- a/lib/deployex/application.ex +++ b/lib/deployex/application.ex @@ -14,6 +14,7 @@ defmodule Deployex.Application do DeployexWeb.Telemetry, Deployex.Storage.Local, Deployex.Monitor.Supervisor, + Deployex.Tracer.Server, {DNSCluster, query: Application.get_env(:deployex, :dns_cluster_query) || :ignore}, {Phoenix.PubSub, name: Deployex.PubSub}, # Start the Finch HTTP client for sending emails diff --git a/lib/deployex/tracer.ex b/lib/deployex/tracer.ex new file mode 100644 index 0000000..80b4361 --- /dev/null +++ b/lib/deployex/tracer.ex @@ -0,0 +1,102 @@ +defmodule Deployex.Tracer do + @moduledoc """ + + + Copied/Modified from + """ + + require Logger + + alias Deployex.Tracer.Server, as: TracerServer + + @default_session_timeout_ms 30_000 + @default_max_msg 5 + + @type t :: %__MODULE__{ + status: :idle | :running, + session_id: binary() | nil, + max_messages: non_neg_integer(), + session_timeout_ms: non_neg_integer(), + functions_by_node: map(), + request_pid: pid() | nil + } + + defstruct status: :idle, + session_id: nil, + max_messages: @default_max_msg, + session_timeout_ms: @default_session_timeout_ms, + functions_by_node: %{}, + request_pid: nil + + ### ========================================================================== + ### Public APIs + ### ========================================================================== + + @doc """ + This function retrieves all modules within the passed node + """ + @spec get_modules(node :: atom()) :: list() + def get_modules(node \\ Node.self()) do + :rpc.call(node, :code, :all_loaded, [], :infinity) + |> Enum.map(fn + {module, _} -> module + module -> module + end) + end + + @doc """ + This function retrieves all functions within the passed node and module + """ + @spec get_module_functions_info(node :: atom(), module :: atom()) :: %{ + functions: map(), + module: atom(), + node: atom() + } + def get_module_functions_info(node \\ Node.self(), module) do + functions = :rpc.call(node, module, :module_info, [:functions], :infinity) + externals = :rpc.call(node, module, :module_info, [:exports], :infinity) + + all_functions = + (functions ++ externals) + |> Enum.reduce(%{}, fn {name, arity}, acc -> + full_name = "#{name}/#{arity}" + + if :erl_internal.guard_bif(name, arity) == false and + regular_functions?(full_name) == false do + Map.put(acc, full_name, %{name: name, arity: arity}) + else + acc + end + end) + + %{node: node, module: module, functions: all_functions} + end + + @spec start_trace(functions :: list(), attrs :: map()) :: + {:ok, t()} | {:error, :already_started} + def start_trace(functions, attrs \\ %{}) do + TracerServer.start_trace(functions, attrs) + end + + @spec stop_trace(binary()) :: :ok + def stop_trace(session_id) do + TracerServer.stop_trace(session_id) + end + + @spec state :: map() + def state do + TracerServer.state() + end + + ### ========================================================================== + ### Private functions + ### ========================================================================== + defp regular_functions?(function) do + String.contains?(function, "-anonymous-") or + String.contains?(function, "-fun-") or + String.contains?(function, "-inlined-") or + String.contains?(function, "-lists^") or + String.contains?(function, "-lc") or + String.contains?(function, "-lbc") + end +end diff --git a/lib/deployex/tracer/server.ex b/lib/deployex/tracer/server.ex new file mode 100644 index 0000000..32954fb --- /dev/null +++ b/lib/deployex/tracer/server.ex @@ -0,0 +1,213 @@ +defmodule Deployex.Tracer.Server do + @moduledoc """ + This server is responsible for handling tracing requests. + + Inspired by: + * https://www.erlang.org/docs/24/man/dbg + * https://github.com/erlang/otp/blob/master/lib/observer/src/observer_trace_wx.erl + * https://kaiwern.com/posts/2020/11/02/debugging-with-tracing-in-elixir/ + * https://blog.appsignal.com/2023/01/10/debugging-and-tracing-in-erlang.html + """ + use GenServer + require Logger + + alias Deployex.Common + alias Deployex.Tracer, as: DeployexT + + ### ========================================================================== + ### GenServer Callbacks + ### ========================================================================== + def start_link(args) do + GenServer.start_link(__MODULE__, args, name: __MODULE__) + end + + @impl true + def init(_args) do + Logger.info("Initializing Tracing Server") + {:ok, %DeployexT{}} + end + + @impl true + def handle_call( + {:start_trace, _functions, _session_timeout_ms}, + _from, + %DeployexT{status: :running} = state + ) do + {:reply, {:error, :already_started}, state} + end + + def handle_call({:stop_tracing, rcv_session_id}, _from, %DeployexT{ + session_id: session_id + }) + when rcv_session_id == session_id do + Logger.info("The Trace session_id: #{inspect(session_id)} was requested to stop.") + + :dbg.stop() + + {:reply, :ok, %DeployexT{}} + end + + def handle_call({:stop_tracing, _rcv_session_id}, _from, state) do + {:reply, :ok, state} + end + + def handle_call(:state, _from, state) do + {:reply, state, state} + end + + def handle_call( + {:start_trace, + %{ + max_messages: max_messages, + session_id: session_id, + request_pid: request_pid, + functions_by_node: functions_by_node, + session_timeout_ms: session_timeout_ms + } = new_state}, + _from, + _state + ) do + Logger.warning( + "New trace requested session: #{session_id} functions: #{inspect(functions_by_node)}" + ) + + tracer_pid = self() + # The local node (deployex) is always present in the trace list of nodes. + # The following list will indicate to the trace handler whether the node + # should be included or filtered out. + monitored_nodes = Map.keys(functions_by_node) + + handle_trace = fn + {_, pid, _, {module, fun, args}, timestamp}, {session_id, index} + when index <= max_messages -> + node = :erlang.node(pid) + + if node in monitored_nodes do + {{y, mm, d}, {h, m, s}} = :calendar.now_to_datetime(timestamp) + arg_list = Enum.map(args, &inspect/1) + + message = + "[#{y}-#{mm}-#{d} #{h}:#{m}:#{s}] (#{inspect(pid)}) #{inspect(module)}.#{fun}(#{Enum.join(arg_list, ", ")})" + + send(request_pid, {:new_trace_message, session_id, node, index, message}) + + if index == max_messages do + send(tracer_pid, {:stop_tracing, session_id}) + send(request_pid, {:stop_tracing, session_id}) + :dbg.stop() + end + + {session_id, index + 1} + else + {session_id, index} + end + + _trace_message, _session_index -> + :dbg.stop() + end + + # Start Tracer with Handler Function + :dbg.tracer(:process, {handle_trace, {session_id, 1}}) + + Enum.each(functions_by_node, fn {node, functions} -> + # Add node to tracing process (exclude local node since it is added by default) + if node != Node.self() do + :dbg.n(node) + end + + # Add functions to be traced + Enum.each(functions, fn function -> + :dbg.tp(function.module, function.function, function.arity, []) + end) + end) + + # :all -> All processes and ports in the system as well as all processes and ports + # created hereafter are to be traced. + # :c -> Traces global function calls for the process according to the trace patterns + # set in the system (see tp/2). + :dbg.p(:all, [:c, :timestamp]) + + Process.send_after(self(), {:trace_session_timeout, session_id}, session_timeout_ms) + + new_state = %{new_state | status: :running} + {:reply, {:ok, new_state}, new_state} + end + + @impl true + def handle_info( + {:trace_session_timeout, session_id_timed_out}, + %DeployexT{session_id: session_id} = state + ) + when session_id != session_id_timed_out do + {:noreply, state} + end + + def handle_info({:trace_session_timeout, rcv_session_id} = msg, %DeployexT{ + session_id: session_id, + request_pid: request_pid + }) + when rcv_session_id == session_id do + Logger.info("The Trace session_id: #{inspect(session_id)} timed out") + + :dbg.stop() + + send(request_pid, msg) + + {:noreply, %DeployexT{}} + end + + def handle_info({:trace_session_timeout, _rcv_session_id}, state) do + {:noreply, state} + end + + def handle_info({:stop_tracing, rcv_session_id} = msg, %DeployexT{ + session_id: session_id, + max_messages: max_messages, + request_pid: request_pid + }) + when rcv_session_id == session_id do + Logger.info("Max messages (#{max_messages}) reached for session: #{inspect(session_id)}.") + + send(request_pid, msg) + + {:noreply, %DeployexT{}} + end + + def handle_info({:stop_tracing, _session_id}, state) do + {:noreply, state} + end + + ### ========================================================================== + ### Public APIs + ### ========================================================================== + + @spec start_trace(functions :: list(), attrs :: map()) :: + {:ok, DeployexT.t()} | {:error, :already_started} + def start_trace(functions, attrs) do + GenServer.call( + __MODULE__, + {:start_trace, + struct( + DeployexT, + attrs + |> Map.put(:request_pid, self()) + |> Map.put(:session_id, Common.uuid4()) + |> Map.put(:functions_by_node, Enum.group_by(functions, & &1.node)) + )} + ) + end + + @spec stop_trace(binary()) :: :ok + def stop_trace(session_id) do + GenServer.call(__MODULE__, {:stop_tracing, session_id}) + end + + @spec state :: DeployexT.t() + def state do + GenServer.call(__MODULE__, :state) + end + + ### ========================================================================== + ### Private Functions + ### ========================================================================== +end diff --git a/lib/deployex_web/components/layouts/app.html.heex b/lib/deployex_web/components/layouts/app.html.heex index 28661ea..b991089 100644 --- a/lib/deployex_web/components/layouts/app.html.heex +++ b/lib/deployex_web/components/layouts/app.html.heex @@ -69,28 +69,49 @@ Live Logs + - - + + + + + - Host Terminal + Live Tracing + Observer + + + + + + + Host Terminal + +
-
<%= item.name %>:
+
<%= item.name %>:
<%= for key <- item.keys do %> diff --git a/lib/deployex_web/live/components/multi_select_list.ex b/lib/deployex_web/live/components/multi_select_list.ex new file mode 100644 index 0000000..04a0a7d --- /dev/null +++ b/lib/deployex_web/live/components/multi_select_list.ex @@ -0,0 +1,158 @@ +defmodule DeployexWeb.Components.MultiSelectList do + @moduledoc """ + Multi select box + + References: + * https://www.creative-tim.com/twcomponents/component/multi-select + + """ + use Phoenix.Component + + alias Phoenix.LiveView.JS + + attr :id, :string, required: true + attr :selected_text, :string, required: true + attr :selected, :list, required: true + attr :unselected, :list, required: true + attr :show_options, :boolean, required: true + + def content(assigns) do + ~H""" +
+
+
+
+
+
+
+ <%= @selected_text %> +
+ + <%= for item <- @selected do %> + <%= for key <- item.keys do %> +
+
+ <%= "#{item.name}:#{key}" %> +
+ +
+ <% end %> + <% end %> +
+ +
+
+
+ +
+
+ +
+
+
+ <%= for item <- @unselected do %> +
+
+
<%= item.name %>:
+
+ + <%= for key <- item.keys do %> + + <% end %> +
+ <% end %> +
+
+
+
+
+
+
+ """ + end +end diff --git a/lib/deployex_web/live/observer/index.ex b/lib/deployex_web/live/observer/index.ex index 9e8a331..6622744 100644 --- a/lib/deployex_web/live/observer/index.ex +++ b/lib/deployex_web/live/observer/index.ex @@ -73,7 +73,7 @@ defmodule DeployexWeb.ObserverLive do %{name: "services", keys: @unselected_services_keys}, %{name: "apps", keys: @unselected_apps_keys} ]} - show_options={@show_apps_options} + show_options={@show_observer_options} /> + + +
+
+ + """ + end + + @impl true + def mount(_params, _session, socket) when is_connected?(socket) do + # Subscribe to notifications if any node is UP or Down + :net_kernel.monitor_nodes(true) + + {:ok, + socket + |> assign(:node_info, update_node_info()) + |> assign(:node_data, %{}) + |> assign(:trace_session_id, nil) + |> assign(:show_tracing_options, false)} + end + + def mount(_params, _session, socket) do + {:ok, + socket + |> assign(:node_info, node_info_new()) + |> assign(:node_data, %{}) + |> assign(:trace_session_id, nil) + |> assign(:show_tracing_options, false)} + end + + @impl true + def handle_params(params, _url, socket) do + {:noreply, apply_action(socket, socket.assigns.live_action, params)} + end + + defp apply_action(socket, :index, _params) do + socket + |> assign(:page_title, "Live Tracing") + end + + @impl true + def handle_event("toggle-options", _value, socket) do + show_tracing_options = !socket.assigns.show_tracing_options + + {:noreply, socket |> assign(:show_tracing_options, show_tracing_options)} + end + + def handle_event( + "tracing-apps-stop", + _data, + %{assigns: %{trace_session_id: trace_session_id}} = socket + ) do + DeployexT.stop_trace(trace_session_id) + {:noreply, assign(socket, :trace_session_id, nil)} + end + + def handle_event("tracing-apps-run", _data, %{assigns: %{node_info: node_info}} = socket) do + tracer_state = DeployexT.state() + + if tracer_state.status == :idle do + functions_to_monitor = + Enum.reduce(node_info.selected_services_keys, [], fn service_key, service_acc -> + service_info = Enum.find(node_info.node, &(&1.service == service_key)) + + service_acc ++ + Enum.reduce(node_info.selected_modules_keys, [], fn module_key, module_acc -> + module_key_atom = String.to_existing_atom(module_key) + + node_functions_info = + Enum.find(service_info.functions, &(&1.module == module_key_atom)) + + functions = + Enum.reduce(node_info.selected_functions_keys, [], fn function_key, + function_acc -> + function = Map.get(node_functions_info.functions, function_key, nil) + + if module_key in service_info.modules_keys and function do + function_acc ++ + [ + %{ + node: String.to_existing_atom(service_key), + module: module_key_atom, + function: function.name, + arity: function.arity + } + ] + else + function_acc + end + end) + + # If the module doesn't have any of the requested functions the default is to + # include the whole module + if functions == [] do + module_acc ++ + [ + %{ + node: String.to_existing_atom(service_key), + module: module_key_atom, + function: :_, + arity: :_ + } + ] + else + module_acc ++ functions + end + end) + end) + + case DeployexT.start_trace(functions_to_monitor) do + {:ok, %{session_id: session_id}} -> + {:noreply, assign(socket, :trace_session_id, session_id)} + + {:error, _} -> + {:noreply, assign(socket, :trace_session_id, nil)} + end + else + {:noreply, assign(socket, :trace_session_id, nil)} + end + end + + def handle_event( + "multi-select-remove-item", + %{"item" => "services", "key" => service_key}, + %{assigns: %{node_info: node_info}} = socket + ) do + node_info = + update_node_info( + node_info.selected_services_keys -- [service_key], + node_info.selected_modules_keys, + node_info.selected_functions_keys + ) + + {:noreply, + socket + |> assign(:node_info, node_info)} + end + + def handle_event( + "multi-select-remove-item", + %{"item" => "modules", "key" => module_key}, + %{assigns: %{node_info: node_info}} = socket + ) do + node_info = + update_node_info( + node_info.selected_services_keys, + node_info.selected_modules_keys -- [module_key], + node_info.selected_functions_keys + ) + + {:noreply, + socket + |> assign(:node_info, node_info)} + end + + def handle_event( + "multi-select-remove-item", + %{"item" => "functions", "key" => function_key}, + %{assigns: %{node_info: node_info}} = socket + ) do + node_info = + update_node_info( + node_info.selected_services_keys, + node_info.selected_modules_keys, + node_info.selected_functions_keys -- [function_key] + ) + + {:noreply, + socket + |> assign(:node_info, node_info)} + end + + def handle_event( + "multi-select-add-item", + %{"item" => "services", "key" => service_key}, + %{assigns: %{node_info: node_info}} = socket + ) do + node_info = + update_node_info( + node_info.selected_services_keys ++ [service_key], + node_info.selected_modules_keys, + node_info.selected_functions_keys + ) + + {:noreply, + socket + |> assign(:node_info, node_info)} + end + + def handle_event( + "multi-select-add-item", + %{"item" => "modules", "key" => module_key}, + %{assigns: %{node_info: node_info}} = socket + ) do + node_info = + update_node_info( + node_info.selected_services_keys, + node_info.selected_modules_keys ++ [module_key], + node_info.selected_functions_keys + ) + + {:noreply, + socket + |> assign(:node_info, node_info)} + end + + def handle_event( + "multi-select-add-item", + %{"item" => "functions", "key" => function_key}, + %{assigns: %{node_info: node_info}} = socket + ) do + node_info = + update_node_info( + node_info.selected_services_keys, + node_info.selected_modules_keys, + node_info.selected_functions_keys ++ [function_key] + ) + + {:noreply, + socket + |> assign(:node_info, node_info)} + end + + @impl true + def handle_info({:nodeup, _node}, %{assigns: %{node_info: node_info}} = socket) do + node_info = + update_node_info( + node_info.selected_services_keys, + node_info.selected_modules_keys, + node_info.selected_functions_keys + ) + + {:noreply, assign(socket, :node_info, node_info)} + end + + def handle_info({:nodedown, node}, %{assigns: %{node_info: node_info}} = socket) do + service_key = node |> to_string + + node_info = + update_node_info( + node_info.selected_services_keys -- [service_key], + node_info.selected_modules_keys, + node_info.selected_functions_keys + ) + + {:noreply, + socket + |> assign(:node_info, node_info)} + end + + def handle_info({:new_trace_message, _session_id, node, index, message}, socket) do + Logger.info("Message: #{node} - [#{index}] :: #{message}") + + {:noreply, socket} + end + + def handle_info({event, _session_id}, socket) + when event in [:trace_session_timeout, :stop_tracing] do + {:noreply, + socket + |> assign(:trace_session_id, nil) + |> assign(:show_tracing_options, false)} + end + + defp node_info_new do + %{ + services_keys: [], + modules_keys: [], + functions_keys: [], + selected_services_keys: [], + selected_modules_keys: [], + selected_functions_keys: [], + node: [] + } + end + + defp update_node_info, do: update_node_info([], [], []) + + defp update_node_info(selected_services_keys, selected_modules_keys, selected_functions_keys) do + initial_map = + %{ + node_info_new() + | selected_services_keys: selected_services_keys, + selected_modules_keys: selected_modules_keys, + selected_functions_keys: selected_functions_keys + } + + Enum.reduce(Node.list() ++ [Node.self()], initial_map, fn instance_node, + %{ + services_keys: services_keys, + modules_keys: modules_keys, + functions_keys: functions_keys, + node: node + } = acc -> + service = instance_node |> to_string + service_selected? = service in selected_services_keys + + [name, _hostname] = String.split(service, "@") + services_keys = (services_keys ++ [service]) |> Enum.sort() + + instance_module_keys = + if service_selected? do + DeployexT.get_modules(instance_node) |> Enum.map(&to_string/1) + else + [] + end + + {instance_functions_keys, functions} = + Enum.reduce(instance_module_keys, {[], []}, fn module, {keys, fun} -> + if module in selected_modules_keys do + module_functions_info = + DeployexT.get_module_functions_info(instance_node, String.to_existing_atom(module)) + + function_keys = Map.keys(module_functions_info.functions) |> Enum.map(&to_string/1) + {keys ++ function_keys, fun ++ [module_functions_info]} + else + {keys, fun} + end + end) + + modules_keys = (modules_keys ++ instance_module_keys) |> Enum.sort() |> Enum.uniq() + functions_keys = (functions_keys ++ instance_functions_keys) |> Enum.sort() |> Enum.uniq() + + node = + if service_selected? do + [ + %{ + name: name, + modules_keys: instance_module_keys, + function_keys: instance_functions_keys, + service: service, + functions: functions + } + | node + ] + else + node + end + + %{ + acc + | services_keys: services_keys, + modules_keys: modules_keys, + functions_keys: functions_keys, + node: node + } + end) + end +end diff --git a/lib/deployex_web/router.ex b/lib/deployex_web/router.ex index 69e1621..1ca47ec 100644 --- a/lib/deployex_web/router.ex +++ b/lib/deployex_web/router.ex @@ -46,6 +46,7 @@ defmodule DeployexWeb.Router do live "/terminal", TerminalLive, :index live "/logs", LogsLive, :index live "/observer", ObserverLive, :index + live "/tracing", TracingLive, :index live "/applications", ApplicationsLive, :index live "/applications/:instance/logs/stdout", ApplicationsLive, :logs_stdout live "/applications/:instance/logs/stderr", ApplicationsLive, :logs_stderr