Implement check-in scheduler

Implement a check-in event scheduler, which schedules check-in events
to be transmitted in a separate Elixir process, with a minimum wait
period of ten seconds between requests, and a wait period of a tenth
of a second before the first request, referred to as "debounce"
periods.

This is a relatively minor improvement for the existing cron
check-ins, but it is a requirement for the heartbeat check-ins, both
to avoid slowing down customers' applications with blocking requests,
and to avoid misuse of the feature from spamming our servers.

The scheduler also acts as a deduplicator, removing "similar enough"
check-in events -- again, not particularly interesting for cron
check-ins, but a requirement to minimise damage when heartbeat
check-ins are misused.

Use the previously implemented support for NDJSON payloads in the
transmitter in order to send all the check-ins in a single request.

Similar to the probes, an Elixir process is started by the AppSignal
application supervisor. When the process receives an event to be
transmitted, it stores it and schedules a message, which will trigger
the transmit, to be sent to itself at a later time.

When this Elixir process receives a shutdown signal, it attempts to
transmit all scheduled events before it shuts down.

.changesets/send-check-ins-concurrently.md

@@ -0,0 +1,6 @@
+---
+bump: patch
+type: change
+---
+
+Send check-ins concurrently. When calling `Appsignal.CheckIn.cron`, instead of blocking the current process while the check-in events are sent, schedule them to be sent in a separate process.

config/config.exs

@@ -7,7 +7,10 @@ if Mix.env() in [:bench, :test, :test_no_nif] do
   config :appsignal, appsignal: Appsignal.FakeAppsignal
   config :appsignal, appsignal_integration_logger: Appsignal.FakeIntegrationLogger
   config :appsignal, appsignal_transmitter: Appsignal.FakeTransmitter
+  config :appsignal, appsignal_checkin_scheduler: Appsignal.FakeScheduler
+  config :appsignal, appsignal_checkin_debounce: Appsignal.FakeDebounce
   config :appsignal, inet: FakeInet
+  config :appsignal, system: FakeSystem
   config :appsignal, io: FakeIO
   config :appsignal, file: FakeFile
   config :appsignal, os_internal: FakeOS

lib/appsignal.ex

@@ -47,7 +47,8 @@ defmodule Appsignal do
     children = [
       {Appsignal.Tracer, []},
       {Appsignal.Monitor, []},
-      {Appsignal.Probes, []}
+      {Appsignal.Probes, []},
+      {Appsignal.CheckIn.Scheduler, []}
     ]
 
     result = Supervisor.start_link(children, strategy: :one_for_one, name: Appsignal.Supervisor)

lib/appsignal/check_in/check_in.ex

@@ -0,0 +1,19 @@
+defmodule Appsignal.CheckIn do
+  alias Appsignal.CheckIn.Cron
+
+  @spec cron(String.t()) :: :ok
+  def cron(identifier) do
+    Cron.finish(Cron.new(identifier))
+  end
+
+  @spec cron(String.t(), (-> out)) :: out when out: var
+  def cron(identifier, fun) do
+    cron = Cron.new(identifier)
+
+    Cron.start(cron)
+    output = fun.()
+    Cron.finish(cron)
+
+    output
+  end
+end

lib/appsignal/check_in/cron.ex

@@ -0,0 +1,64 @@
+defmodule Appsignal.CheckIn.Cron do
+  alias __MODULE__
+  alias Appsignal.CheckIn.Cron.Event
+
+  @scheduler Application.compile_env(
+               :appsignal,
+               :appsignal_checkin_scheduler,
+               Appsignal.CheckIn.Scheduler
+             )
+  @type t :: %Cron{identifier: String.t(), digest: String.t()}
+
+  defstruct [:identifier, :digest]
+
+  @spec new(String.t()) :: t
+  def new(identifier) do
+    %Cron{
+      identifier: identifier,
+      digest: random_digest()
+    }
+  end
+
+  defp random_digest do
+    Base.encode16(:crypto.strong_rand_bytes(8), case: :lower)
+  end
+
+  @spec start(Cron.t()) :: :ok
+  def start(cron) do
+    @scheduler.schedule(Event.new(cron, :start))
+  end
+
+  @spec finish(Cron.t()) :: :ok
+  def finish(cron) do
+    @scheduler.schedule(Event.new(cron, :finish))
+  end
+end
+
+defmodule Appsignal.CheckIn.Cron.Event do
+  alias __MODULE__
+  alias Appsignal.CheckIn.Cron
+
+  @derive Jason.Encoder
+
+  @type kind :: :start | :finish
+  @type t :: %Event{
+          identifier: String.t(),
+          digest: String.t(),
+          kind: kind,
+          timestamp: integer,
+          check_in_type: :cron
+        }
+
+  defstruct [:identifier, :digest, :kind, :timestamp, :check_in_type]
+
+  @spec new(Cron.t(), kind) :: t
+  def new(%Cron{identifier: identifier, digest: digest}, kind) do
+    %Event{
+      identifier: identifier,
+      digest: digest,
+      kind: kind,
+      timestamp: System.system_time(:second),
+      check_in_type: :cron
+    }
+  end
+end