May 30, 2019

Elixir resources
to help with transition to Elixir

We are planning introducing Elixir into our toolbox. This page summarizes key resources we have user / are using for learning Elixir and pushing it to production. Feel free to propose changes via pull-request.

Deployment & Containers/Kubernetes

Motivation is to be able to deploy apps leveraging OTP to k8s (and running in containers). Especially important piece of having a support for OTP is to be able to use things like long-running GenServer processes, migrate state etc. Valuable resources for this topic are

This leads into the following key building blocks:

Distributed systems / data-types

  • Using Rust to Scale Elixir for 11 Million Concurrent Users
    • Rust implementation of SortedSet which is then used by Elixir backend
  • An Adventure in Distributed Programming by Wiebe-Marten Wijnja
  • Building Resilient Systems with Stacking by Chris Keathley

    • Recording from ElixrConf EU 2019
    • Overview of techniques which helps in building more resilient systems. Refers to How Complex Systems Fail for parallels between medical systems and complex distributed services.
    • Circuit brakers: Recommended implementation is fuse.
    • Configuration: Should avoid use of “mix configs”, instead he pointed to (his) project Vapor. Example of usage (from the talk, chech project for other one):

      defmodule Jenga.Application do
        use Application
      
        def start(_type, _args) do
          config = [
              port: "PORT",
              db_url: "DB_URL",
          ]
      
          children = [
              {Jenga.Config, config},
          ]
      
          opts = [strategy: :one_for_one, name: Jenga.Supervisor]
          Supervisor.start_link(children, opts)
        end
      end
      
      defmodule Jenga.Config do
          use GenServer
      
          def start_link(desired_config) do
              GenServer.start_link(__MODULE__, desired_config, name: __MODULE__)
          end
      
          def init(desired) do
              :jenga_config = :ets.new(:jenga_config, [:set, :protected, :named_table])
              case load_config(:jenga_config, desired) do
              :ok ->
                  {:ok, %{table: :jenga_config, desired: desired}}
              :error ->
                  {:stop, :could_not_load_config}
              end
          end
      
          defp load_config(table, config, retry_count \\ 0)
          defp load_config(_table, [], _), do: :ok
          defp load_config(_table, _, 10), do: :error
          defp load_config(table, [{k, v} | tail], retry_count) do
              case System.get_env(v) do
              nil ->
                  load_config(table, [{k, v} | tail], retry_count + 1)
              value ->
                  :ets.insert(table, {k, value})
                  load_config(table, tail, retry_count)
              end
          end
      end
      
    • Monitoring: you can use Erlang’s alarms. Example from the talk, which takes database as dependency and if not reachable will raise an alarm:

      defmodule Jenga.Database.Watchdog do
      use GenServer
      
      def init(:ok) do
          schedule_check()
          {:ok, %{status: :degraded, passing_checks: 0}}
      end
      
      def handle_info(:check_db, state) do
          status = Jenga.Database.check_status()
          state = change_state(status, state)
          schedule_check()
          {:noreply, state}
      end
      
      defp change_state(result, %{status: status, passing_checks: count}) do
          case {result, status, count} do
          {:ok, :connected, count} ->
              if count == 3 do
              :alarm_handler.clear_alarm(@alarm_id)
              end
              %{status: :connected, passing_checks: count + 1}
      
          {:ok, :degraded, _} ->
              %{status: :connected, passing_checks: 0}
      
          {:error, :connected, _} ->
              :alarm_handler.set_alarm({@alarm_id, "We cannot connect to the database”})
              %{status: :degraded, passing_checks: 0}
              {:error, :degraded, _} ->
                  %{status: :degraded, passing_checks: 0}
          end
      end
      end
      

      Then alarm handle can be added:

      defmodule Jenga.Application do
        use Application
      
        def start(_type, _args) do
          config = [
              port: "PORT",
              db_url: "DB_URL",
          ]
      
          :gen_event.swap_handler(
              :alarm_handler,
              {:alarm_handler, :swap},
              {Jenga.AlarmHandler, :ok}
          )
      
          children = [
              {Jenga.Config, config},
              Jenga.Database.Supervisor,
          ]
      
          opts = [strategy: :one_for_one, name: Jenga.Supervisor]
          Supervisor.start_link(children, opts)
        end
      end
      
      defmodule Jenga.AlarmHandler do
        require Logger
      
        def init({:ok, {:alarm_handler, _old_alarms}}) do
          Logger.info("Installing alarm handler")
          {:ok %{}}
        end
      
        def handle_event({:set_alarm, :database_disconnected}, alarms) do
          # Do something with the alarm rising (e.g. notify monitoring)
          Logger.error("Database connection lost")
          {:ok, alarms}
        end
      
        def handle_event({:clear_alarm, :database_disconnected}, alarms) do
          # Do something with the alarm being cleared (e.g. notify monitoring)
          Logger.error("Database connection recovered")
          {:ok, alarms}
        end
      
        def handle_event(event, state) do
          Logger.info("Unhandled alarm event: #{inspect(event)}")
          {:ok, state}
        end
      end
      
      

Best practices

Code/Development

  • typical suspect - credo

    mix credo --strict
    
  • spend time writing documentation in code with ExDoc

    • write typespecs as they are pulled by ExDoc but also used by tools like dialyzer
    • deploy dialyzer from the very beginning of the project
  • use official formatter in your projects

    mix format --check-formatted
    
  • nice writeup about putting these tools together - https://itnext.io/enforcing-code-quality-in-elixir-20f87efc7e66

Code Design

Ops/Infrastructure/Monitoring