Get alerted in Slack or by email when an Upsun cron or backup fails

Upsun runs every cron as an environment.cron activity and every backup as an environment.backup activity, and both carry a result of success or failure. Nothing on the platform notifies you when that result is a failure.

Both ways of noticing are manual: run the activity command in the CLI, or open the Console activity list — where cron activities are filtered out by default, so you have to tick the Crons box before they even appear. A cron can fail for days before anyone looks. Automated backups are worse: they can fail for months and be discovered missing exactly when you need one.

Here is an activity script that closes the gap. It covers crons and backups from one table of activity types, sends to Slack and optionally email, and sends one alert per failure rather than one per run — a cron on */5 would otherwise produce 288 messages a day. A reminder follows after 24 hours if it is still failing, and one message when it recovers.

Installing it

upsun integration:add --project <PROJECT_ID> \
  --type script \
  --file ./failure-alert.js \
  --events=environment.cron,environment.backup \
  --states=complete

Configuration comes from activity script variables, set per integration and readable only from inside the script — so nothing secret goes into the file and it can be committed as it stands. Set them with POST /projects/<PROJECT_ID>/integrations/<INTEGRATION_ID>/variables, marking the webhook and the shared secret is_sensitive: true: the value still reaches the script, but a GET of the variable omits it, so it cannot be read back out of the API. There is no CLI subcommand for these as of 5.13.0 — it is API-only, via upsun api:curl.

Three things that cost me time

1. The Storage API is a CommonJS module, not a global — require('storage'). Without it every call throws ReferenceError, and because the script catches its own errors the integration still reports complete / success while the anti-flood is completely dead.
2. There is no getPrimaryRoute() in the runtime. Use activity.payload.environment.edge_hostname.
3. environment.backup covers both scheduled backups and ones a person starts — there is no separate activity type for automated backups.

Because the script catches its own errors, the integration's activity always reports complete / success even when nothing was sent. Read the log, not the state: upsun integration:activity:log <INTEGRATION_ID> <ACTIVITY_ID> (the integration ID is positional, not an option). An empty log is a real result — it means the script ran and deliberately stayed silent.

The script

'use strict';

// The Storage API is a CommonJS module, NOT a global.
const storage = require('storage');

/* Configuration comes from ACTIVITY SCRIPT VARIABLES, set per integration and
   readable only from inside the script. Nothing secret lives in this file, so
   it can be committed as it stands:

     POST /projects/<PROJECT_ID>/integrations/<INTEGRATION_ID>/variables
     {"name": "SLACK_WEBHOOK_URL", "value": "https://hooks.slack.com/...",
      "is_sensitive": true}

   With is_sensitive the value still reaches the script, but a GET of the
   variable omits it, so it cannot be read back out of the API. */
const SLACK_WEBHOOK_URL = readVariable('SLACK_WEBHOOK_URL');
const ALERT_MAIL_ENDPOINT_PATH = readVariable('ALERT_MAIL_ENDPOINT_PATH');
const ALERT_MAIL_SECRET = readVariable('ALERT_MAIL_SECRET');
const ALERT_MAIL_TO = readVariable('ALERT_MAIL_TO');

/* `variables` is a global, but it is absent on older runtimes and empty when
   no variable is set, so read it defensively. */
function readVariable(name) {
  try {
    const value = variables[name];
    return typeof value === 'string' ? value : '';
  } catch (error) {
    return '';
  }
}

const REMINDER_INTERVAL_MS = 24 * 60 * 60 * 1000;
const LOG_TAIL_LINES = 20;

/* The activity types this script reacts to. Both carry a `result` of
   `success` or `failure` and are reported in the `complete` state, which is
   what lets one script serve both. Anything else is ignored.

   `environment.backup` covers BOTH scheduled backups and ones started by a
   person: there is no separate type for automated backups. A backup has no
   per-run name worth keying on (its `backup_name` is unique to each backup,
   so keying on it would make every run look new), hence `keySuffix: null`. */
const ACTIVITY_TYPES = {
  'environment.cron': {
    noun: 'cron',
    field: 'Cron',
    label: function (activity) { return 'cron ' + cronName(activity); },
    keySuffix: cronName
  },
  'environment.backup': {
    noun: 'backup',
    field: 'Backup',
    label: function () { return 'backup'; },
    keySuffix: function () { return null; }
  }
};

function isConfigured(value) {
  return typeof value === 'string' && value !== '';
}

main();

function main() {
  const descriptor = ACTIVITY_TYPES[activity && activity.type];
  if (!descriptor) {
    return;                      // misconfigured event filter: stay quiet
  }

  const label = descriptor.label(activity);
  const environmentName = getEnvironmentName();
  const key = storageKey(descriptor, environmentName);
  const stored = readState(key);
  const now = Date.now();
  const result = activity.result;

  if (result === 'success') {
    if (!stored || stored.state !== 'failing') {
      return;                    // healthy and stayed healthy: do nothing
    }
    if (dispatch('Upsun ' + label + ' recovered',
                 buildRecovery(descriptor, label, environmentName, stored, now), 'good')) {
      writeState(key, { state: 'ok', since: now, failures: 0, lastReminder: 0 });
    } else {
      console.log('failure-alert: recovery undelivered; state left as failing');
    }
    return;
  }

  if (result !== 'failure') {
    return;                      // an unexpected result value
  }

  if (!stored || stored.state !== 'failing') {
    // Healthy -> failing. This is the alert.
    if (dispatch('Upsun FAILED: ' + label,
                 buildFailure(descriptor, label, environmentName), 'danger')) {
      writeState(key, { state: 'failing', since: now, failures: 1, lastReminder: now });
    } else {
      // Nothing was sent, so do not record the transition: the next failing
      // run retries the alert.
      console.log('failure-alert: alert undelivered; state not recorded');
    }
    return;
  }

  // Still failing. Stay silent until the reminder window elapses.
  const failures = (stored.failures || 0) + 1;
  const lastReminder = stored.lastReminder || stored.since || now;

  if (now - lastReminder < REMINDER_INTERVAL_MS) {
    writeState(key, { state: 'failing', since: stored.since || now,
                      failures: failures, lastReminder: lastReminder });
    return;
  }

  const sent = dispatch('Upsun STILL failing: ' + label,
    buildReminder(descriptor, label, environmentName, stored.since || now, failures, now), 'danger');
  writeState(key, {
    state: 'failing',
    since: stored.since || now,
    failures: failures,
    lastReminder: sent ? now : lastReminder   // only move it if it went out
  });
}

/* ---- Reading the activity ---------------------------------------- */

function cronName(activity) {
  const payload = activity.payload || {};
  if (payload.cron && payload.cron.name) return payload.cron.name;
  if (activity.parameters && activity.parameters.cron) return activity.parameters.cron;
  return 'unknown';
}

function getEnvironmentName() {
  const payload = activity.payload || {};
  if (payload.environment && payload.environment.name) return payload.environment.name;
  if (Array.isArray(activity.environments) && activity.environments.length) {
    return activity.environments[0];
  }
  return 'unknown-env';
}

/* One state document per project + environment + type (+ cron name), so no
   two things ever share one piece of state. */
function storageKey(descriptor, environmentName) {
  const projectId = (project && project.id) || 'unknown';
  const suffix = descriptor.keySuffix(activity);
  return 'failure-alert:' + projectId + ':' + environmentName + ':' +
    descriptor.noun + (suffix ? ':' + suffix : '');
}

/* There is no getPrimaryRoute(). The edge hostname is carried on the
   activity payload, and is present with or without a custom domain. */
function getAppBaseUrl() {
  const environment = (activity.payload && activity.payload.environment) || {};
  return environment.edge_hostname ? 'https://' + environment.edge_hostname : '';
}

function getConsoleUrl() {
  const projectId = (project && project.id) || '';
  return projectId ? 'https://console.upsun.com/projects/' + projectId : '';
}

function getLogTail() {
  const log = typeof activity.log === 'string' ? activity.log : '';
  if (log === '') return '(no log output)';
  return log.replace(/\s+$/, '').split('\n').slice(-LOG_TAIL_LINES).join('\n');
}

function getProjectName() {
  return (project && (project.title || project.id)) || 'unknown project';
}

function formatDuration(ms) {
  const totalMinutes = Math.max(0, Math.round(ms / 60000));
  const hours = Math.floor(totalMinutes / 60);
  const minutes = totalMinutes % 60;
  return hours === 0 ? minutes + ' minute(s)'
                     : hours + ' hour(s) ' + minutes + ' minute(s)';
}

/* ---- Message bodies ----------------------------------------------- */

function buildFailure(descriptor, label, environmentName) {
  return [
    'Project:     ' + getProjectName(),
    'Environment: ' + environmentName,
    descriptor.field + ':' + (descriptor.field.length < 8 ? '        ' : '   ') + label,
    'Completed:   ' + (activity.completed_at || new Date().toISOString()),
    'Console:     ' + getConsoleUrl(),
    '',
    'Last ' + LOG_TAIL_LINES + ' log lines:',
    getLogTail()
  ].join('\n');
}

function buildReminder(descriptor, label, environmentName, since, failures, now) {
  return [
    'This ' + descriptor.noun + ' has been failing for ' + formatDuration(now - since) + '.',
    failures + ' run(s) have failed since ' + new Date(since).toISOString() + '.',
    '',
    buildFailure(descriptor, label, environmentName)
  ].join('\n');
}

function buildRecovery(descriptor, label, environmentName, stored, now) {
  const since = stored.since || now;
  return [
    'Project:     ' + getProjectName(),
    'Environment: ' + environmentName,
    '',
    'The ' + descriptor.noun + ' is succeeding again after ' + formatDuration(now - since) +
      ' and ' + (stored.failures || 0) + ' failed run(s).',
    'Console:     ' + getConsoleUrl()
  ].join('\n');
}

/* ---- State -------------------------------------------------------- */

function readState(key) {
  try {
    const raw = storage.get(key);
    if (!raw) return null;
    return typeof raw === 'string' ? JSON.parse(raw) : raw;
  } catch (error) {
    console.log('failure-alert: could not read state: ' + error);
    return null;
  }
}

function writeState(key, value) {
  try {
    storage.set(key, JSON.stringify(value));   // string-safe values only
  } catch (error) {
    console.log('failure-alert: could not write state: ' + error);
  }
}

/* ---- Delivery ------------------------------------------------------ */

/* Returns true when at least one channel accepted the message. Never
   throws: a broken webhook must not corrupt the stored state. */
function dispatch(title, body, colour) {
  let delivered = false;
  if (sendSlack(title, body, colour)) delivered = true;
  if (sendEmail(title, body)) delivered = true;
  return delivered;
}

function sendSlack(title, body, colour) {
  if (!isConfigured(SLACK_WEBHOOK_URL)) return false;
  return post('Slack', SLACK_WEBHOOK_URL, { 'Content-Type': 'application/json' }, {
    text: title,
    attachments: [{
      color: colour,            // 'danger' renders the red bar
      title: title,
      text: body,
      footer: 'Upsun activity script',
      ts: Math.floor(Date.now() / 1000)
    }]
  });
}

function sendEmail(subject, body) {
  if (!isConfigured(ALERT_MAIL_TO)) return false;
  if (!isConfigured(ALERT_MAIL_ENDPOINT_PATH) || !isConfigured(ALERT_MAIL_SECRET)) {
    return false;
  }

  const base = getAppBaseUrl();
  if (!base) {
    console.log('failure-alert: no edge hostname on the payload; skipping email');
    return false;
  }

  const url = base.replace(/\/+$/, '') + '/' + ALERT_MAIL_ENDPOINT_PATH.replace(/^\/+/, '');
  return post('alert-mail endpoint', url, {
    'Content-Type': 'application/json',
    'X-Alert-Secret': ALERT_MAIL_SECRET
  }, { subject: subject, body: body, to: ALERT_MAIL_TO });
}

/* One synchronous POST. fetch returns the response itself, not a promise. */
function post(channel, url, headers, payload) {
  try {
    const response = fetch(url, {
      method: 'POST',
      headers: headers,
      body: JSON.stringify(payload)
    });
    if (!response || response.ok === false || (response.status && response.status >= 400)) {
      console.log('failure-alert: ' + channel + ' rejected with status ' +
        ((response && response.status) || 'unknown'));
      return false;
    }
    return true;
  } catch (error) {
    console.log('failure-alert: ' + channel + ' threw: ' + error);
    return false;
  }
}

Verified end to end on a live project with CLI 5.13.0: alert on failure, silence on the repeat, one message on recovery.