diff --git a/bigbluebutton-html5/imports/api/connection-status/server/methods/voidConnection.js b/bigbluebutton-html5/imports/api/connection-status/server/methods/voidConnection.js
index 85297dbab7..7f369c5a3d 100644
--- a/bigbluebutton-html5/imports/api/connection-status/server/methods/voidConnection.js
+++ b/bigbluebutton-html5/imports/api/connection-status/server/methods/voidConnection.js
@@ -1,4 +1,9 @@
+import { PrometheusAgent, METRIC_NAMES } from '/imports/startup/server/prom-metrics/index.js'
+
// Round-trip time helper
-export default function voidConnection() {
+export default function voidConnection(previousRtt) {
+ if (previousRtt) {
+ PrometheusAgent.observe(METRIC_NAMES.METEOR_RTT, previousRtt/1000);
+ }
return 0;
}
diff --git a/bigbluebutton-html5/imports/startup/server/index.js b/bigbluebutton-html5/imports/startup/server/index.js
index eadeb94cd8..6cc7653bbe 100755
--- a/bigbluebutton-html5/imports/startup/server/index.js
+++ b/bigbluebutton-html5/imports/startup/server/index.js
@@ -144,7 +144,8 @@ Meteor.startup(() => {
Meteor.onMessage(event => {
const { method } = event;
if (method) {
- PrometheusAgent.increment(METRIC_NAMES.METEOR_METHODS, { methodName: method });
+ const methodName = method.includes('stream-cursor') ? 'stream-cursor' : method;
+ PrometheusAgent.increment(METRIC_NAMES.METEOR_METHODS, { methodName });
}
});
diff --git a/bigbluebutton-html5/imports/startup/server/logger.js b/bigbluebutton-html5/imports/startup/server/logger.js
index 6896a7c6f6..7f7502502c 100755
--- a/bigbluebutton-html5/imports/startup/server/logger.js
+++ b/bigbluebutton-html5/imports/startup/server/logger.js
@@ -1,5 +1,6 @@
import { Meteor } from 'meteor/meteor';
import { createLogger, format, transports } from 'winston';
+import WinstonPromTransport from './prom-metrics/winstonPromTransport';
const LOG_CONFIG = Meteor?.settings?.private?.serverLog || {};
const { level } = LOG_CONFIG;
@@ -20,6 +21,10 @@ const Logger = createLogger({
handleExceptions: true,
level,
}),
+ // export error logs to prometheus
+ new WinstonPromTransport({
+ level: 'error',
+ }),
],
});
diff --git a/bigbluebutton-html5/imports/startup/server/prom-metrics/metrics.js b/bigbluebutton-html5/imports/startup/server/prom-metrics/metrics.js
index d5a7b7c83a..903cbe5230 100644
--- a/bigbluebutton-html5/imports/startup/server/prom-metrics/metrics.js
+++ b/bigbluebutton-html5/imports/startup/server/prom-metrics/metrics.js
@@ -1,34 +1,60 @@
const {
Counter,
+ Gauge,
+ Histogram
} = require('prom-client');
const METRICS_PREFIX = 'html5_'
const METRIC_NAMES = {
METEOR_METHODS: 'meteorMethods',
+ METEOR_ERRORS_TOTAL: 'meteorErrorsTotal',
+ METEOR_RTT: 'meteorRtt',
+ REDIS_MESSAGE_QUEUE: 'redisMessageQueue',
+ REDIS_PAYLOAD_SIZE: 'redisPayloadSize',
+ REDIS_PROCESSING_TIME: 'redisProcessingTime'
}
-const buildFrontendMetrics = () => {
- return {
- [METRIC_NAMES.METEOR_METHODS]: new Counter({
- name: `${METRICS_PREFIX}meteor_methods`,
- help: 'Total number of meteor methods processed in html5',
- labelNames: ['methodName', 'role', 'instanceId'],
- }),
- }
-}
-
-const buildBackendMetrics = () => {
- // TODO add relevant backend metrics
- return {}
-}
-
let METRICS;
const buildMetrics = () => {
if (METRICS == null) {
- const isFrontend = (!process.env.BBB_HTML5_ROLE || process.env.BBB_HTML5_ROLE === 'frontend');
- const isBackend = (!process.env.BBB_HTML5_ROLE || process.env.BBB_HTML5_ROLE === 'backend');
- if (isFrontend) METRICS = buildFrontendMetrics();
- if (isBackend) METRICS = { ...METRICS, ...buildBackendMetrics()}
+ METRICS = {
+ [METRIC_NAMES.METEOR_METHODS]: new Counter({
+ name: `${METRICS_PREFIX}meteor_methods`,
+ help: 'Total number of meteor methods processed in html5',
+ labelNames: ['methodName', 'role', 'instanceId'],
+ }),
+
+ [METRIC_NAMES.METEOR_ERRORS_TOTAL]: new Counter({
+ name: `${METRICS_PREFIX}meteor_errors_total`,
+ help: 'Total number of errors logs in meteor',
+ labelNames: ['errorMessage', 'role', 'instanceId'],
+ }),
+
+ [METRIC_NAMES.METEOR_RTT]: new Histogram({
+ name: `${METRICS_PREFIX}meteor_rtt_seconds`,
+ help: 'Round-trip time of meteor client-server connections in seconds',
+ buckets: [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1, 1.5, 2, 2.5, 5],
+ labelNames: ['role', 'instanceId'],
+ }),
+
+ [METRIC_NAMES.REDIS_MESSAGE_QUEUE]: new Gauge({
+ name: `${METRICS_PREFIX}redis_message_queue`,
+ help: 'Message queue size in redis',
+ labelNames: ['meetingId', 'role', 'instanceId'],
+ }),
+
+ [METRIC_NAMES.REDIS_PAYLOAD_SIZE]: new Histogram({
+ name: `${METRICS_PREFIX}redis_payload_size`,
+ help: 'Redis events payload size',
+ labelNames: ['eventName', 'role', 'instanceId'],
+ }),
+
+ [METRIC_NAMES.REDIS_PROCESSING_TIME]: new Histogram({
+ name: `${METRICS_PREFIX}redis_processing_time`,
+ help: 'Redis events processing time in milliseconds',
+ labelNames: ['eventName', 'role', 'instanceId'],
+ }),
+ }
}
return METRICS;
diff --git a/bigbluebutton-html5/imports/startup/server/prom-metrics/promAgent.js b/bigbluebutton-html5/imports/startup/server/prom-metrics/promAgent.js
index 6c0ab53140..b7c668a808 100644
--- a/bigbluebutton-html5/imports/startup/server/prom-metrics/promAgent.js
+++ b/bigbluebutton-html5/imports/startup/server/prom-metrics/promAgent.js
@@ -81,6 +81,16 @@ class PrometheusScrapeAgent {
metric.set(labelsObject, value)
}
}
+
+ observe(metricName, value, labelsObject) {
+ if (!this.started) return;
+
+ const metric = this.metrics[metricName];
+ if (metric) {
+ labelsObject = { ...labelsObject, ...this.roleAndInstanceLabels };
+ metric.observe(labelsObject, value)
+ }
+ }
}
export default PrometheusScrapeAgent;
diff --git a/bigbluebutton-html5/imports/startup/server/prom-metrics/winstonPromTransport.js b/bigbluebutton-html5/imports/startup/server/prom-metrics/winstonPromTransport.js
new file mode 100644
index 0000000000..44669b199a
--- /dev/null
+++ b/bigbluebutton-html5/imports/startup/server/prom-metrics/winstonPromTransport.js
@@ -0,0 +1,19 @@
+const Transport = require('winston-transport');
+import { PrometheusAgent, METRIC_NAMES } from './index.js'
+
+module.exports = class WinstonPromTransport extends Transport {
+ constructor(opts) {
+ super(opts);
+
+ }
+
+ log(info, callback) {
+ setImmediate(() => {
+ this.emit('logged', info);
+ });
+
+ PrometheusAgent.increment(METRIC_NAMES.METEOR_ERRORS_TOTAL, { errorMessage: info.message });
+
+ callback();
+ }
+};
diff --git a/bigbluebutton-html5/imports/startup/server/redis.js b/bigbluebutton-html5/imports/startup/server/redis.js
index c6035359a7..a80472b9b5 100755
--- a/bigbluebutton-html5/imports/startup/server/redis.js
+++ b/bigbluebutton-html5/imports/startup/server/redis.js
@@ -5,11 +5,13 @@ import { check } from 'meteor/check';
import Logger from './logger';
import Metrics from './metrics';
import queue from 'queue';
+import { PrometheusAgent, METRIC_NAMES } from './prom-metrics/index.js'
// Fake meetingId used for messages that have no meetingId
const NO_MEETING_ID = '_';
const { queueMetrics } = Meteor.settings.private.redis.metrics;
+const { collectRedisMetrics: PROM_METRICS_ENABLED } = Meteor.settings.private.prometheus;
const makeEnvelope = (channel, eventName, header, body, routing) => {
const envelope = {
@@ -78,6 +80,16 @@ class MeetingMessageQueue {
}
const queueLength = this.queue.length;
+
+ if (PROM_METRICS_ENABLED) {
+ const dataLength = JSON.stringify(data).length;
+ const currentTimestamp = Date.now();
+ const processTime = currentTimestamp - beginHandleTimestamp;
+ PrometheusAgent.observe(METRIC_NAMES.REDIS_PROCESSING_TIME, processTime, { eventName });
+ PrometheusAgent.observe(METRIC_NAMES.REDIS_PAYLOAD_SIZE, dataLength, { eventName });
+ meetingId && PrometheusAgent.set(METRIC_NAMES.REDIS_MESSAGE_QUEUE, queueLength, { meetingId });
+ }
+
if (queueLength > 100) {
Logger.warn(`Redis: MeetingMessageQueue for meetingId=${meetingId} has queue size=${queueLength} `);
}
diff --git a/bigbluebutton-html5/imports/ui/components/connection-status/service.js b/bigbluebutton-html5/imports/ui/components/connection-status/service.js
index 237cce1b58..1105af48c5 100644
--- a/bigbluebutton-html5/imports/ui/components/connection-status/service.js
+++ b/bigbluebutton-html5/imports/ui/components/connection-status/service.js
@@ -29,6 +29,7 @@ const intlMessages = defineMessages({
});
let stats = -1;
+let lastRtt = null;
const statsDep = new Tracker.Dependency();
let statsTimeout = null;
@@ -111,11 +112,12 @@ const addConnectionStatus = (level, type, value) => {
const fetchRoundTripTime = () => {
const t0 = Date.now();
- makeCall('voidConnection').then(() => {
+ makeCall('voidConnection', lastRtt).then(() => {
const tf = Date.now();
const rtt = tf - t0;
const event = new CustomEvent('socketstats', { detail: { rtt } });
window.dispatchEvent(event);
+ lastRtt = rtt;
});
};
diff --git a/bigbluebutton-html5/private/config/settings.yml b/bigbluebutton-html5/private/config/settings.yml
index fda73e7e3c..cf7db32eaa 100755
--- a/bigbluebutton-html5/private/config/settings.yml
+++ b/bigbluebutton-html5/private/config/settings.yml
@@ -833,3 +833,5 @@ private:
path: '/metrics'
# Whether default metrics for Node.js processes should be exported
collectDefaultMetrics: false
+ # Whether redis metrics should be exported
+ collectRedisMetrics: false