Merge pull request #14781 from germanocaumo/more-prom-metrics

refactor(prom-html5) More prometheus metrics

bigbluebutton-html5/imports/api/connection-status/server/methods/voidConnection.js

@@ -1,4 +1,9 @@
+import { PrometheusAgent, METRIC_NAMES } from '/imports/startup/server/prom-metrics/index.js'
+
 // Round-trip time helper
-export default function voidConnection() {
+export default function voidConnection(previousRtt) {
+  if (previousRtt) {
+    PrometheusAgent.observe(METRIC_NAMES.METEOR_RTT, previousRtt/1000);
+  }
   return 0;
 }

bigbluebutton-html5/imports/startup/server/index.js

@@ -144,7 +144,8 @@ Meteor.startup(() => {
   Meteor.onMessage(event => {
     const { method } = event;
     if (method) {
-      PrometheusAgent.increment(METRIC_NAMES.METEOR_METHODS, { methodName: method });
+      const methodName = method.includes('stream-cursor') ? 'stream-cursor' : method;
+      PrometheusAgent.increment(METRIC_NAMES.METEOR_METHODS, { methodName });
     }
   });
 

bigbluebutton-html5/imports/startup/server/logger.js

@@ -1,5 +1,6 @@
 import { Meteor } from 'meteor/meteor';
 import { createLogger, format, transports } from 'winston';
+import WinstonPromTransport from './prom-metrics/winstonPromTransport';
 
 const LOG_CONFIG = Meteor?.settings?.private?.serverLog || {};
 const { level } = LOG_CONFIG;
@@ -20,6 +21,10 @@ const Logger = createLogger({
       handleExceptions: true,
       level,
     }),
+    // export error logs to prometheus
+    new WinstonPromTransport({
+      level: 'error',
+    }),
   ],
 });
 

bigbluebutton-html5/imports/startup/server/prom-metrics/metrics.js

@@ -1,34 +1,60 @@
 const {
   Counter,
+  Gauge,
+  Histogram
 } = require('prom-client');
 
 const METRICS_PREFIX = 'html5_'
 const METRIC_NAMES = {
   METEOR_METHODS: 'meteorMethods',
-}
-
-const buildFrontendMetrics = () => {
-  return {
-    [METRIC_NAMES.METEOR_METHODS]: new Counter({
-      name: `${METRICS_PREFIX}meteor_methods`,
-      help: 'Total number of meteor methods processed in html5',
-      labelNames: ['methodName', 'role', 'instanceId'],
-    }),
-  }
-}
-
-const buildBackendMetrics = () => {
-  // TODO add relevant backend metrics
-  return {}
+  METEOR_ERRORS_TOTAL: 'meteorErrorsTotal',
+  METEOR_RTT: 'meteorRtt',
+  REDIS_MESSAGE_QUEUE: 'redisMessageQueue',
+  REDIS_PAYLOAD_SIZE: 'redisPayloadSize',
+  REDIS_PROCESSING_TIME: 'redisProcessingTime'
 }
 
 let METRICS;
 const buildMetrics = () => {
   if (METRICS == null) {
-    const isFrontend = (!process.env.BBB_HTML5_ROLE || process.env.BBB_HTML5_ROLE === 'frontend');
-    const isBackend = (!process.env.BBB_HTML5_ROLE || process.env.BBB_HTML5_ROLE === 'backend');
-    if (isFrontend) METRICS = buildFrontendMetrics();
-    if (isBackend) METRICS = { ...METRICS, ...buildBackendMetrics()} 
+    METRICS = {
+      [METRIC_NAMES.METEOR_METHODS]: new Counter({
+        name: `${METRICS_PREFIX}meteor_methods`,
+        help: 'Total number of meteor methods processed in html5',
+        labelNames: ['methodName', 'role', 'instanceId'],
+      }),
+
+      [METRIC_NAMES.METEOR_ERRORS_TOTAL]: new Counter({
+        name: `${METRICS_PREFIX}meteor_errors_total`,
+        help: 'Total number of errors logs in meteor',
+        labelNames: ['errorMessage', 'role', 'instanceId'],
+      }),
+
+      [METRIC_NAMES.METEOR_RTT]: new Histogram({
+        name: `${METRICS_PREFIX}meteor_rtt_seconds`,
+        help: 'Round-trip time of meteor client-server connections in seconds',
+        buckets: [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1, 1.5, 2, 2.5, 5],
+        labelNames: ['role', 'instanceId'],
+      }),
+
+      [METRIC_NAMES.REDIS_MESSAGE_QUEUE]: new Gauge({
+        name: `${METRICS_PREFIX}redis_message_queue`,
+        help: 'Message queue size in redis',
+        labelNames: ['meetingId', 'role', 'instanceId'],
+      }),
+
+      [METRIC_NAMES.REDIS_PAYLOAD_SIZE]: new Histogram({
+        name: `${METRICS_PREFIX}redis_payload_size`,
+        help: 'Redis events payload size',
+        labelNames: ['eventName', 'role', 'instanceId'],
+      }),
+
+      [METRIC_NAMES.REDIS_PROCESSING_TIME]: new Histogram({
+        name: `${METRICS_PREFIX}redis_processing_time`,
+        help: 'Redis events processing time in milliseconds',
+        labelNames: ['eventName', 'role', 'instanceId'],
+      }),
+    }
   }
 
   return METRICS;

bigbluebutton-html5/imports/startup/server/prom-metrics/promAgent.js

@@ -81,6 +81,16 @@ class PrometheusScrapeAgent {
       metric.set(labelsObject, value)
     }
   }
+
+  observe(metricName, value, labelsObject) {
+    if (!this.started) return;
+
+    const metric = this.metrics[metricName];
+    if (metric) {
+      labelsObject = { ...labelsObject, ...this.roleAndInstanceLabels };
+      metric.observe(labelsObject, value)
+    }
+  }
 }
 
 export default PrometheusScrapeAgent;

bigbluebutton-html5/imports/startup/server/prom-metrics/winstonPromTransport.js

@@ -0,0 +1,19 @@
+const Transport = require('winston-transport');
+import { PrometheusAgent, METRIC_NAMES } from './index.js'
+
+module.exports = class WinstonPromTransport extends Transport {
+  constructor(opts) {
+    super(opts);
+    
+  }
+
+  log(info, callback) {
+    setImmediate(() => {
+      this.emit('logged', info);
+    });
+
+    PrometheusAgent.increment(METRIC_NAMES.METEOR_ERRORS_TOTAL, { errorMessage: info.message });
+
+    callback();
+  }
+};

bigbluebutton-html5/imports/startup/server/redis.js

@@ -5,11 +5,13 @@ import { check } from 'meteor/check';
 import Logger from './logger';
 import Metrics from './metrics';
 import queue from 'queue';
+import { PrometheusAgent, METRIC_NAMES } from './prom-metrics/index.js'
 
 // Fake meetingId used for messages that have no meetingId
 const NO_MEETING_ID = '_';
 
 const { queueMetrics } = Meteor.settings.private.redis.metrics;
+const { collectRedisMetrics: PROM_METRICS_ENABLED  } = Meteor.settings.private.prometheus;
 
 const makeEnvelope = (channel, eventName, header, body, routing) => {
   const envelope = {
@@ -78,6 +80,16 @@ class MeetingMessageQueue {
       }
 
       const queueLength = this.queue.length;
+
+      if (PROM_METRICS_ENABLED) {
+        const dataLength = JSON.stringify(data).length;
+        const currentTimestamp = Date.now();
+        const processTime = currentTimestamp - beginHandleTimestamp;
+        PrometheusAgent.observe(METRIC_NAMES.REDIS_PROCESSING_TIME, processTime, { eventName });
+        PrometheusAgent.observe(METRIC_NAMES.REDIS_PAYLOAD_SIZE, dataLength, { eventName });
+        meetingId && PrometheusAgent.set(METRIC_NAMES.REDIS_MESSAGE_QUEUE, queueLength, { meetingId });
+      }
+
       if (queueLength > 100) {
         Logger.warn(`Redis: MeetingMessageQueue for meetingId=${meetingId} has queue size=${queueLength} `);
       }

bigbluebutton-html5/imports/ui/components/connection-status/service.js

@@ -29,6 +29,7 @@ const intlMessages = defineMessages({
 });
 
 let stats = -1;
+let lastRtt = null;
 const statsDep = new Tracker.Dependency();
 
 let statsTimeout = null;
@@ -111,11 +112,12 @@ const addConnectionStatus = (level, type, value) => {
 
 const fetchRoundTripTime = () => {
   const t0 = Date.now();
-  makeCall('voidConnection').then(() => {
+  makeCall('voidConnection', lastRtt).then(() => {
     const tf = Date.now();
     const rtt = tf - t0;
     const event = new CustomEvent('socketstats', { detail: { rtt } });
     window.dispatchEvent(event);
+    lastRtt = rtt;
   });
 };
 

bigbluebutton-html5/private/config/settings.yml

@@ -833,3 +833,5 @@ private:
     path: '/metrics'
     # Whether default metrics for Node.js processes should be exported
     collectDefaultMetrics: false
+    # Whether redis metrics should be exported
+    collectRedisMetrics: false