Mini Kabibi Habibi
/*************************************************************************
*
* ADOBE CONFIDENTIAL
* ___________________
*
* Copyright 2017 Adobe Systems Incorporated
* All Rights Reserved.
*
* NOTICE: All information contained herein is, and remains
* the property of Adobe Systems Incorporated and its suppliers,
* if any. The intellectual and technical concepts contained
* herein are proprietary to Adobe Systems Incorporated and its
* suppliers and are protected by trade secret or copyright law.
* Dissemination of this information or reproduction of this material
* is strictly forbidden unless prior written permission is obtained
* from Adobe Systems Incorporated.
**************************************************************************/
/*global module, require, console, window */
/**
Utility functions
**/
function generateGUID() {
var s4 = function () {
return Math.floor((1 + Math.random()) * 0x10000)
.toString(16)
.substring(1);
};
return s4() + s4() + '-' + s4() + '-' + s4() + '-' + s4() + '-' + s4() + s4() + s4();
}
function truncateEventQueue(queue, maxLength) {
var truncatedQueue = queue;
if (queue && queue.length > maxLength && maxLength > 0) {
var startIndex = queue.length - maxLength;
truncatedQueue = queue.slice(startIndex, queue.length);
}
return truncatedQueue;
}
function pad(n, length) {
var str = n.toString();
if (str.length < length) {
var padding = [];
padding.length = length - str.length + 1;
str = padding.join('0') + str;
}
return str;
}
function extend(dest, from) {
var props = Object.getOwnPropertyNames(from);
props.forEach(function (name) {
if (typeof from[name] === 'object') {
if (typeof dest[name] !== 'object') {
dest[name] = {};
}
extend(dest[name], from[name]);
} else {
var destination = Object.getOwnPropertyDescriptor(from, name);
Object.defineProperty(dest, name, destination);
}
});
return dest;
}
function notifyCallbacks(callbacks, err, numSentEvents) {
callbacks.forEach(function (callback) {
// Call each callback in a timeout, so if there's an exception in one callback it doesn't affect any others
setTimeout(function () {
callback(err, numSentEvents);
});
});
}
/**
Constants
**/
var LOG_PREFIX = 'Ingest :: ';
var ANALYTICS_HOST = {
prod: 'cc-api-data.adobe.io',
stage: 'cc-api-data-stage.adobe.io',
dev: 'cc-api-data-dev.adobe.io'
};
var INGEST_PATH = '/ingest';
var RETRY_RANDOM_SECONDS = 10;
// Settable options, with their default values
var DEFAULT_OPTIONS = {
ENVIRONMENT: 'prod',
ALLOW_NO_TOKEN: false,
ANALYTICS_INGEST_TYPE: 'dunamis',
ANALYTICS_MAX_QUEUED_EVENTS: 50,
ANALYTICS_DEBOUNCE: 10000,
ANALYTICS_API_KEY: null,
ANALYTICS_X_PRODUCT: null,
ANALYTICS_X_PRODUCT_LOCATION: undefined,
ANALYTICS_PROJECT: null,
ANALYTICS_USER_REGION: 'UNKNOWN',
TIMESTAMP_PROPERTY_NAME: 'event.dts_end'
};
var REQUIRED_OPTIONS = [
'ANALYTICS_API_KEY',
'ANALYTICS_X_PRODUCT',
'ANALYTICS_PROJECT',
];
/**
Ingest Class
**/
// Constructor
function Ingest(dependencies, options) {
dependencies = dependencies || {};
options = options || {};
var throwError = message => {
this._log(message);
throw new Error('ERROR: ' + message);
};
// Internal state
this._queuedEvents = [];
this._queuedCallbacks = [];
this._lastSendTime = 0;
this._isEnabled = false; // Sending analytics is disabled by default
// Configure dependencies
this._dependencies = extend({}, dependencies);
if (!dependencies.getAccessToken || typeof dependencies.getAccessToken !== 'function') {
throwError('Missing dependency: getAccessToken');
}
// Configure options
this._options = {};
Object.keys(DEFAULT_OPTIONS).forEach(key => {
this._options[key] = options[key] || DEFAULT_OPTIONS[key];
});
// Make sure required options have been passed in
REQUIRED_OPTIONS.forEach(option => {
if (!this._options[option]) {
throwError('Missing option: ' + option);
}
});
// Make sure we have fetch
if (typeof fetch === 'undefined') {
throwError(`Ingest requires fetch - if in a node environment, set 'global.fetch = require('node-fetch');'`);
}
}
Ingest.prototype._log = function (message) {
var doLog = this._dependencies.log;
if (doLog) {
doLog(LOG_PREFIX + message);
}
};
Ingest.prototype._getAgent = function (url, callback) {
if (this._dependencies.getAgent) {
this._dependencies.getAgent(url, callback);
return;
}
callback(null, {});
};
Ingest.prototype._getAccessToken = function (callback) {
this._dependencies.getAccessToken(callback);
};
Ingest.prototype._clearAccessToken = function () {
if (this._dependencies.clearAccessToken) {
this._dependencies.clearAccessToken();
}
};
Ingest.prototype._getEnvironment = function () {
return ANALYTICS_HOST[this._options.ENVIRONMENT] ? this._options.ENVIRONMENT : 'prod';
};
Ingest.prototype._getAnalyticsHost = function () {
return ANALYTICS_HOST[this._getEnvironment()];
};
Ingest.prototype._formatTimestamp = function (date) {
// Corresponds to moment format string 'YYYY-MM-DDTHH:mm:ss.SSSZZ'
var YYYY = date.getFullYear();
var MM = pad(date.getMonth() + 1, 2); // Month is 0-11
var DD = pad(date.getDate(), 2);
var HH = pad(date.getHours(), 2);
var mm = pad(date.getMinutes(), 2);
var ss = pad(date.getSeconds(), 2);
var SSS = pad(date.getMilliseconds(), 3);
var offset = date.getTimezoneOffset();
var sign = offset < 0 ? '+' : '-'; // Sign is inverted
var hours = Math.floor(Math.abs(offset) / 60);
var mins = Math.abs(offset) % 60;
var ZZ = sign + pad(hours, 2) + pad(mins, 2);
return YYYY + '-' + MM + '-' + DD + 'T' + HH + ':' + mm + ':' + ss + '.' + SSS + ZZ;
};
Ingest.prototype._updateDebounce = function (headers) {
var retryAfterHeader = headers && (headers['retry-after'] || headers['Retry-After']);
var retryAfter = 0;
if (retryAfterHeader) {
var retryTime;
try {
// First, try to parse it as a number (retry time in seconds)
retryTime = parseInt(retryAfterHeader, 10);
} catch (ignore) {
// ignore
}
if (retryTime) {
retryAfter = Math.max(0, retryTime);
} else {
// If that fails, try to parse it as a date
var retryDate = Date.parse(retryAfterHeader);
if (retryDate) {
// Need to add a randomised element to ensure requests don't all come back at the same time
var now = new Date().valueOf();
var retrySeconds = Math.max(0, retryDate - now) / 1000;
var retryRandom = Math.floor(Math.random() * RETRY_RANDOM_SECONDS);
retryAfter = retrySeconds + retryRandom;
}
}
}
this._options.ANALYTICS_DEBOUNCE = Math.max(retryAfter * 1000, this._options.ANALYTICS_DEBOUNCE);
};
Ingest.prototype._queueEvent = function (event) {
if (this._queuedEvents.length >= this._options.ANALYTICS_MAX_QUEUED_EVENTS) {
this._queuedEvents.shift();
}
this._queuedEvents.push(event);
};
Ingest.prototype._requeueEvents = function (failedEvents) {
// If we failed sending events, add them back to the beginning of the queue - but make sure it doesn't go over the maximum length
this._queuedEvents = failedEvents.concat(this._queuedEvents);
this._queuedEvents = truncateEventQueue(this._queuedEvents, this._options.ANALYTICS_MAX_QUEUED_EVENTS);
};
Ingest.prototype._sendAnalytics = function (sendImmediately, callback, retryAttemps) {
retryAttemps = retryAttemps || 0;
if (callback) {
this._queuedCallbacks.push(callback);
}
if (!this._isEnabled || this._queuedEvents.length === 0) {
var callbacks = this._queuedCallbacks;
this._queuedCallbacks = [];
if (!this._isEnabled) {
notifyCallbacks(callbacks, new Error('Analytics Disabled'));
} else {
notifyCallbacks(callbacks, null, 0);
}
return;
}
var debounce = this._options.ANALYTICS_DEBOUNCE;
if (sendImmediately) {
// Clear any timeout, and set the debounce to zero, to force an immediate send
debounce = 0;
clearTimeout(this._pendingSendAnalyticsTimeout);
this._pendingSendAnalyticsTimeout = undefined;
}
if (this._sendingEvents || this._pendingSendAnalyticsTimeout) {
// We're in the middle of sending analytics already
// This will automatically kick off another send afterwards, so no need to do anything
// However, we'd want to store if we need to kick off another send with `sendImmediately = true` afterwards.
if (sendImmediately) {
this._fastFollow = true;
}
return;
}
var currentTime = new Date().valueOf();
if (currentTime - this._lastSendTime < debounce) {
// Throttle analytics, so we don't send too often - this allows us to batch up analytics
this._pendingSendAnalyticsTimeout = setTimeout(() => {
this._pendingSendAnalyticsTimeout = undefined;
this._sendAnalytics();
}, debounce);
return;
}
this._lastSendTime = currentTime;
// The queued events are now going to be sent
this._sendingEvents = this._queuedEvents;
this._sendingCallbacks = this._queuedCallbacks;
this._queuedEvents = [];
this._queuedCallbacks = [];
var requestId = generateGUID();
var logPrefix = '[' + requestId + '] ';
var ingestData = {
events: this._sendingEvents
};
// This gets called when finished, whether we got a response or failed with an error
var onFinished = err => {
var numNewEvents = this._queuedEvents ? this._queuedEvents.length : 0;
if (this._sendingEvents) {
var numSentEvents = this._sendingEvents.length;
if (err) {
this._requeueEvents(this._sendingEvents);
this._log(logPrefix + 'Error sending ' + numSentEvents + ' events: ' + err);
} else {
this._log(logPrefix + 'Success sending ' + numSentEvents + ' events: ' + JSON.stringify(this._sendingEvents));
}
delete this._sendingEvents;
var sendingCallbacks = this._sendingCallbacks;
delete this._sendingCallbacks;
if (err) {
notifyCallbacks(sendingCallbacks, err);
} else {
notifyCallbacks(sendingCallbacks, null, numSentEvents);
}
}
// If there were any new events while sending the last batch, trigger another send.
// Note: This doesn't auto-trigger a retry if we failed, and there were no new events.
if (numNewEvents > 0) {
// fastFollow stores whether a client has called us with sendImmediately = true when we were busy
this._sendAnalytics(this._fastFollow);
}
this._fastFollow = false;
};
// This gets called when we get an actual response from the server
var handleResponse = (statusCode, headers) => {
this._updateDebounce(headers);
if (statusCode === 401 && retryAttemps === 0) {
this._clearAccessToken();
this._requeueEvents(this._sendingEvents);
delete this._sendingEvents;
this._queuedCallbacks = this._sendingCallbacks.concat(this._queuedCallbacks);
delete this._sendingCallbacks;
// Retry one more time
this._log(logPrefix + 'Access token is expired. Retry one more time.');
this._sendAnalytics(true, undefined, retryAttemps + 1);
return;
}
if (statusCode !== 200) {
onFinished(new Error('Unexpected Response: ' + statusCode));
return;
}
onFinished();
};
this._getAccessToken((err, token) => {
if (err && !this._options.ALLOW_NO_TOKEN) {
onFinished(err);
return;
}
if ((!token || token.length === 0) && !this._options.ALLOW_NO_TOKEN) {
onFinished(new Error('No access token'));
return;
}
var urlBase = 'https://' + this._getAnalyticsHost();
this._log(logPrefix + 'Sending analytics to ' + urlBase + INGEST_PATH);
const headers = {
'x-api-key': this._options.ANALYTICS_API_KEY,
'X-Product': this._options.ANALYTICS_X_PRODUCT,
'User-Agent': this._options.ANALYTICS_USER_AGENT || this._options.ANALYTICS_API_KEY,
'X-Request-Id': requestId,
'Content-Type': 'application/json'
};
if (token) {
headers.Authorization = 'Bearer ' + token;
}
if (this._options.ANALYTICS_X_PRODUCT_LOCATION) {
headers['X-Product-Location'] = this._options.ANALYTICS_X_PRODUCT_LOCATION;
}
this._getAgent(urlBase, (err, proxyOptions) => {
const options = {
method: 'POST',
headers,
body: JSON.stringify(ingestData)
};
if (proxyOptions && proxyOptions.agent) {
options.agent = proxyOptions && proxyOptions.agent;
} else {
extend(options, proxyOptions || {});
}
fetch(urlBase + INGEST_PATH, options).then(response => {
handleResponse(response.status, response.headers);
}, onFinished);
});
});
};
/**
Public APIs
**/
/**
* Configure whether analytics is enabled or not. Note: By default, analytics are disabled, so you need to
* explicitly call `ingest.enable(true)` to enable sending analytics.
*
* When sending analytics is disabled, events are still queued up, so they can be sent when it's reenabled.
*
* @param {Boolean} isEnabled Whether to enable or disable sending analytics.
*
* @memberof Ingest
*/
Ingest.prototype.enable = function (isEnabled) {
this._isEnabled = isEnabled;
if (isEnabled) {
// If we enable analytics, trigger flushing any queued events
this._sendAnalytics(true);
}
};
/**
* Post an analytics event to ingest.
*
* @param {Object} payload Ingest payload to be sent
* @param {Function} [callback] If supplied, called when the event has been posted (or failed)
*
* @memberof Ingest
*/
Ingest.prototype.postEvent = function (payload, callback, options) {
var overrideOptions = options || {};
var dtsStart = 'event.dts_start';
var collDts = 'event.coll_dts';
var dtsEnd = overrideOptions.TIMESTAMP_PROPERTY_NAME || this._options.TIMESTAMP_PROPERTY_NAME;
var ingestProject = overrideOptions.ANALYTICS_PROJECT || this._options.ANALYTICS_PROJECT;
var ingestType = overrideOptions.ANALYTICS_INGEST_TYPE || this._options.ANALYTICS_INGEST_TYPE;
if (payload[collDts] && payload[collDts] instanceof Date) {
payload[collDts] = this._formatTimestamp(payload[collDts]);
}
if (payload[dtsStart] && payload[dtsStart] instanceof Date) {
payload[dtsStart] = this._formatTimestamp(payload[dtsStart]);
}
// Set to current time, if it's not supplied
if (!payload[dtsEnd]) {
payload[dtsEnd] = this._formatTimestamp(new Date());
}
if (payload[dtsEnd] instanceof Date) {
payload[dtsEnd] = this._formatTimestamp(payload[dtsEnd]);
}
var event = {
time: payload[dtsEnd],
project: ingestProject,
environment: this._getEnvironment(),
ingesttype: ingestType,
data: payload
};
/**
* Payload's 'simulate' property triggers the option to not
* actually post the event but dump all relevant data to a log
*/
if (!payload.simulate) {
this._queueEvent(event);
this._sendAnalytics(false, callback);
} else {
this._log('event sim:' + JSON.stringify(event));
if (callback) {
notifyCallbacks([ callback ], null, 0);
}
}
};
/**
* Flush the analytics (trigger sending any queued analytics to the server)
*
* @param {Boolean} immediate Flush event immediately or not
* @param {Function} [callback] If supplied, called when the flush has finished (or failed)
*
* @memberof Ingest
*/
Ingest.prototype.flush = function (sendImmediately, callback) {
this._sendAnalytics(sendImmediately, callback);
};
module.exports = Ingest;