function giveNameToAnonymousFunction () {
		var node = this;

		if (node[0].name == "var" || node[0].name == "object") {
			node[1].forEach(function (assignemt) {
				if (assignemt[1]) {
					if (assignemt[1][0].name === "function") {
						assignemt[1][0].anonymousName = assignemt[0];
					} else if (assignemt[1][0].name === "conditional") {
						if (assignemt[1][2][0] && assignemt[1][2][0].name === "function") {
							assignemt[1][2][0].anonymousName = assignemt[0];
						}
						if (assignemt[1][3][0] && assignemt[1][3][0].name === "function") {
							assignemt[1][3][0].anonymousName = assignemt[0];
						}
					}
				}
			});
		} else if (node[0].name == "assign" && node[1] === true) {
			if (node[3][0].name === "function") {
				node[3][0].anonymousName = getNameFromAssign(node);
			} else if (node[3][0] === "conditional") {
				if (node[3][2][0] && node[3][2][0].name === "function") {
					node[3][2][0].anonymousName = getNameFromAssign(node);
				}
				if (node[3][3][0] && node[3][3][0].name === "function") {
					node[3][3][0].anonymousName = getNameFromAssign(node);
				}
			}
		}
	}

function giveNameToAnonymousFunction ( ) { var node = this ; if ( node [ 0 ] . name == "var" || node [ 0 ] . name == "object" ) { node [ 1 ] . forEach ( function ( assignemt ) { if ( assignemt [ 1 ] ) { if ( assignemt [ 1 ] [ 0 ] . name === "function" ) { assignemt [ 1 ] [ 0 ] . anonymousName = assignemt [ 0 ] ; } else if ( assignemt [ 1 ] [ 0 ] . name === "conditional" ) { if ( assignemt [ 1 ] [ 2 ] [ 0 ] && assignemt [ 1 ] [ 2 ] [ 0 ] . name === "function" ) { assignemt [ 1 ] [ 2 ] [ 0 ] . anonymousName = assignemt [ 0 ] ; } if ( assignemt [ 1 ] [ 3 ] [ 0 ] && assignemt [ 1 ] [ 3 ] [ 0 ] . name === "function" ) { assignemt [ 1 ] [ 3 ] [ 0 ] . anonymousName = assignemt [ 0 ] ; } } } } ) ; } else if ( node [ 0 ] . name == "assign" && node [ 1 ] === true ) { if ( node [ 3 ] [ 0 ] . name === "function" ) { node [ 3 ] [ 0 ] . anonymousName = getNameFromAssign ( node ) ; } else if ( node [ 3 ] [ 0 ] === "conditional" ) { if ( node [ 3 ] [ 2 ] [ 0 ] && node [ 3 ] [ 2 ] [ 0 ] . name === "function" ) { node [ 3 ] [ 2 ] [ 0 ] . anonymousName = getNameFromAssign ( node ) ; } if ( node [ 3 ] [ 3 ] [ 0 ] && node [ 3 ] [ 3 ] [ 0 ] . name === "function" ) { node [ 3 ] [ 3 ] [ 0 ] . anonymousName = getNameFromAssign ( node ) ; } } } }

This function tries to extract the name of anonymous functions depending on where they are
defined.

For instance
var something = function () {}
the function itself is anonymous but we can use 'something' as its name

'node' is anything that gets counted, function are extracted from

var
node[0] : node description
node[1] : array of assignments
node[x][0] : variable name
node[x][1] : value, node

object  (when functions are properties of an object)
node[0] : node description
node[1] : array of attributes
node[x][0] : attribute name
node[x][1] : value, node

assign  (things like object.name = function () {})
node[0] : node description
node[1] : type of assignment, 'true' if '=' or operand (like += |= and others)
node[2] : left value, object property or variable
node[3] : right value, node

in case of assign, node[2] can be
'name' if we assign to a variable
name[0] : node description
name[1] : variable's name
'dot' when we assign to an object's property
dot[0] : node description
dot[1] : container object
dot[2] : property

This function tries to extract the name of anonymous functions depending on where they are defined .


function wrapConditionals () {
		if (options.condition === false) {
			// condition coverage is disabled
			return;
		}

		var self = this, ret;
		if (self[0].start && analyzing.indexOf(self) < 0) {
			analyzing.push(self);
			var lineId = self[0].name + ':' + (self[0].start.line + lineOffset);

			self[1] = wrapCondition(self[1], lineId);

			self[2] = walker.walk(self[2]);
			self[3] = walker.walk(self[3]);

			analyzing.pop(self);

			return self;
		} else if (self[1]) {
			self[1] = wrapCondition(self[1], lineId);
		}
	}

function wrapConditionals ( ) { if ( options . condition === false ) { return ; } var self = this , ret ; if ( self [ 0 ] . start && analyzing . indexOf ( self ) < 0 ) { analyzing . push ( self ) ; var lineId = self [ 0 ] . name + ':' + ( self [ 0 ] . start . line + lineOffset ) ; self [ 1 ] = wrapCondition ( self [ 1 ] , lineId ) ; self [ 2 ] = walker . walk ( self [ 2 ] ) ; self [ 3 ] = walker . walk ( self [ 3 ] ) ; analyzing . pop ( self ) ; return self ; } else if ( self [ 1 ] ) { self [ 1 ] = wrapCondition ( self [ 1 ] , lineId ) ; } }

This function wraps ternary conditionals in order to have condition coverage

'this' is a node containing
'this[0]' node descriptor
'this[1]' decision block
'this[2]' first statement
'this[3]' second statement


function (name, deps, module) {
    switch (arguments.length) {
        case 1: // define(function () {})
            module = name;
            deps = name = sb.undefined;
            break;

        case 2: // define(['a', 'b'], function () {})
            module = deps;
            deps = name;
            name = sb.undefined;
            break;

        case 3: // define('name', ['a', 'b'], function () {})
    }

    if (typeof module !== "function") {
        amdModules[currentModule] = module;
        return;
    }

    var output = {'exports': {}};
    if (!deps) {
        deps = ["require", "exports", "module"];
    }
    for (var i = 0, c = deps.length; i < c; i++) {
        switch (deps[i]) {
            case "require":
                deps[i] = currentRequire;
                break;
            case "module":
                deps[i] = output;
                break;
            case "exports":
                deps[i] = output.exports;
                break;
            default:
                deps[i] = currentRequire && currentRequire(deps[i]);
        }
    }
    module = module.apply(this, deps) || output.exports;
    amdModules[currentModule] = module;
}

function ( name , deps , module ) { switch ( arguments . length ) { case 1 : module = name ; deps = name = sb . undefined ; break ; case 2 : module = deps ; deps = name ; name = sb . undefined ; break ; case 3 : } if ( typeof module !== "function" ) { amdModules [ currentModule ] = module ; return ; } var output = { 'exports' : { } } ; if ( ! deps ) { deps = [ "require" , "exports" , "module" ] ; } for ( var i = 0 , c = deps . length ; i < c ; i ++ ) { switch ( deps [ i ] ) { case "require" : deps [ i ] = currentRequire ; break ; case "module" : deps [ i ] = output ; break ; case "exports" : deps [ i ] = output . exports ; break ; default : deps [ i ] = currentRequire && currentRequire ( deps [ i ] ) ; } } module = module . apply ( this , deps ) || output . exports ; amdModules [ currentModule ] = module ; }

RequireJS & AMD-style define

(function (require) {
var define = require.define;

define(["a"], function (a) {
return a + 2;
});
})

@param name
@param deps
@param module


function stats_calculate_coverage(moduleName) {
    var stats = sb.trigger('*:stats-get', moduleName, null)[1],
        total,
        covered,
        lineId,
        lineNum,
        parts;

    var lineReport = {};

    if (!stats.lines) {
        return;
    }
    stats.coverage = {};

    covered = 0;
    total = stats.lines.length;
    for (lineId in stats.runLines) {
        if (stats.runLines[lineId] > 0) {
            covered++;
        } else {
            lineNum = lineId;
            if (!lineReport[lineNum]) {
                lineReport[lineNum] = {};
            }
            lineReport[lineNum].lines = false;
        }
    }
    stats.coverage.lines = {
        total: total,
        covered: covered,
        percentage: 100.0 * (total ? covered / total : 1)
    };

    covered = 0;
    total = stats.functions.length;
    for (lineId in stats.runFunctions) {
        if (stats.runFunctions[lineId] > 0) {
            covered++;
        } else {
            parts = lineId.split(':');
            lineNum = parts[1];
            if (!lineReport[lineNum]) {
                lineReport[lineNum] = {};
            }
            if (!lineReport[lineNum].functions) {
                lineReport[lineNum].functions = [];
            }
            lineReport[lineNum].functions.push(parts[0]);
        }
    }
    stats.coverage.functions = {
        total:total,
        covered:covered,
        percentage:100.0 * (total ? covered / total : 1)
    };

    covered = 0;
    total = stats.conditions.length;
    for (lineId in stats.runConditions) {
        if (stats.runConditions[lineId][1] > 0) {
            covered += 1;
        }

        if (stats.runConditions[lineId][1] === 0) {

            parts = lineId.split(':');
            lineNum = parts[1];
            if (!lineReport[lineNum]) {
                lineReport[lineNum] = {};
            }
            if (!lineReport[lineNum].conditions) {
                lineReport[lineNum].conditions = [];
            }
            lineReport[lineNum].conditions.push(stats.runConditions[lineId]);
        }
    }
    stats.coverage.conditions = {
        total:total,
        covered:covered,
        percentage:100.0 * (total ? covered / total : 1)
    };
    stats.coverage.report = lineReport;
}

function stats_calculate_coverage ( moduleName ) { var stats = sb . trigger ( '*:stats-get' , moduleName , null ) [ 1 ] , total , covered , lineId , lineNum , parts ; var lineReport = { } ; if ( ! stats . lines ) { return ; } stats . coverage = { } ; covered = 0 ; total = stats . lines . length ; for ( lineId in stats . runLines ) { if ( stats . runLines [ lineId ] > 0 ) { covered ++ ; } else { lineNum = lineId ; if ( ! lineReport [ lineNum ] ) { lineReport [ lineNum ] = { } ; } lineReport [ lineNum ] . lines = false ; } } stats . coverage . lines = { total : total , covered : covered , percentage : 100.0 * ( total ? covered / total : 1 ) } ; covered = 0 ; total = stats . functions . length ; for ( lineId in stats . runFunctions ) { if ( stats . runFunctions [ lineId ] > 0 ) { covered ++ ; } else { parts = lineId . split ( ':' ) ; lineNum = parts [ 1 ] ; if ( ! lineReport [ lineNum ] ) { lineReport [ lineNum ] = { } ; } if ( ! lineReport [ lineNum ] . functions ) { lineReport [ lineNum ] . functions = [ ] ; } lineReport [ lineNum ] . functions . push ( parts [ 0 ] ) ; } } stats . coverage . functions = { total : total , covered : covered , percentage : 100.0 * ( total ? covered / total : 1 ) } ; covered = 0 ; total = stats . conditions . length ; for ( lineId in stats . runConditions ) { if ( stats . runConditions [ lineId ] [ 1 ] > 0 ) { covered += 1 ; } if ( stats . runConditions [ lineId ] [ 1 ] === 0 ) { parts = lineId . split ( ':' ) ; lineNum = parts [ 1 ] ; if ( ! lineReport [ lineNum ] ) { lineReport [ lineNum ] = { } ; } if ( ! lineReport [ lineNum ] . conditions ) { lineReport [ lineNum ] . conditions = [ ] ; } lineReport [ lineNum ] . conditions . push ( stats . runConditions [ lineId ] ) ; } } stats . coverage . conditions = { total : total , covered : covered , percentage : 100.0 * ( total ? covered / total : 1 ) } ; stats . coverage . report = lineReport ; }

Calculate coverage total

@param moduleName


function (config, mixins) {
    if (Array.isArray(config.mixins) && Array.isArray(mixins)) {
        config.mixins.push.apply(config.mixins, mixins);
        return config;
    }

    return deepDestructableMerge(config, {
        mixins: mixins
    });
}

function ( config , mixins ) { if ( Array . isArray ( config . mixins ) && Array . isArray ( mixins ) ) { config . mixins . push . apply ( config . mixins , mixins ) ; return config ; } return deepDestructableMerge ( config , { mixins : mixins } ) ; }

Merges mixins with config

@param {Object} config
@param {String[]} mixins


function (left, right) {
    for (var prop in right) {
        if (right.hasOwnProperty(prop))  {
            if (typeof left[prop] === "object") {
                deepDestructableMerge(left[prop], right[prop]);
            } else {
                left[prop] = right[prop];
            }
        }
    }
    return left;
}

function ( left , right ) { for ( var prop in right ) { if ( right . hasOwnProperty ( prop ) ) { if ( typeof left [ prop ] === "object" ) { deepDestructableMerge ( left [ prop ] , right [ prop ] ) ; } else { left [ prop ] = right [ prop ] ; } } } return left ; }

Config files deep merge

@param {Object} left
@param {Object} right


function (config, configDir) {
    config = config || {};
    if (typeof config.extends !== "string") {
        return config;
    }

    var parentConfig = tryExtend(readConfig(configDir, config.extends), configDir);

    return deepDestructableMerge(parentConfig, config);
}

function ( config , configDir ) { config = config || { } ; if ( typeof config . extends !== "string" ) { return config ; } var parentConfig = tryExtend ( readConfig ( configDir , config . extends ) , configDir ) ; return deepDestructableMerge ( parentConfig , config ) ; }

Merges all config files in module's lineage

@param {Object} config
@param {String} configDir

Merges all config files in module s lineage


function (modulePath, dependsFileMask) {
    modulePath = [].concat(modulePath);

    return modulePath.map(function (modulePath) {
        var fileName = modulePath.replace(/^.*\/|\.[a-z0-9]+$/g, '');

        return path.join(path.dirname(modulePath), dependsFileMask.replace('*', fileName));
    });
}

function ( modulePath , dependsFileMask ) { modulePath = [ ] . concat ( modulePath ) ; return modulePath . map ( function ( modulePath ) { var fileName = modulePath . replace ( / ^.*\/|\.[a-z0-9]+$ / g , '' ) ; return path . join ( path . dirname ( modulePath ) , dependsFileMask . replace ( '*' , fileName ) ) ; } ) ; }

Returns depends config file of this module

@param {String|Array} modulePath
@param {String}       dependsFileMask

@return {Array}


function (configA, configB, flagsNames, isMasterConfig) {
    // Apply Flags
    flagsNames.forEach(function (optionsName) {
        // if master -> B
        if (typeof configB[optionsName] === "undefined") {
            return;
        }

        if (isMasterConfig) {
            configA[optionsName] = configB[optionsName];
        } else {
            // if A literal B array -> B
            if (configB[optionsName] instanceof Array && !(configA[optionsName] instanceof Array) ) {
                configA[optionsName] = configB[optionsName];
            } else if (configB[optionsName] instanceof Array && configA[optionsName] instanceof Array) {
                // if A array B array -> A concat B
                configA[optionsName] = configA[optionsName].concat(configB[optionsName]);
            } else {
                // if A literal B literal -> union
                // if A array B literal -> A
                configA[optionsName] = configA[optionsName] || configB[optionsName];
            }
            // else {}
        }
    });
}

function ( configA , configB , flagsNames , isMasterConfig ) { flagsNames . forEach ( function ( optionsName ) { if ( typeof configB [ optionsName ] === "undefined" ) { return ; } if ( isMasterConfig ) { configA [ optionsName ] = configB [ optionsName ] ; } else { if ( configB [ optionsName ] instanceof Array && ! ( configA [ optionsName ] instanceof Array ) ) { configA [ optionsName ] = configB [ optionsName ] ; } else if ( configB [ optionsName ] instanceof Array && configA [ optionsName ] instanceof Array ) { configA [ optionsName ] = configA [ optionsName ] . concat ( configB [ optionsName ] ) ; } else { configA [ optionsName ] = configA [ optionsName ] || configB [ optionsName ] ; } } } ) ; }

Merges configs flags

@param {Object}   configA
@param {Object}   configB
@param {String[]} flagsNames
@param {Boolean}  isMasterConfig


function addPluginsFromBundles(resultConfig) {
    if (resultConfig.bundles) {
        var bundles = Object.keys(resultConfig.bundles),
            lmdPlugins = Object.keys(LMD_PLUGINS);

        // Apply flags from bundles
        bundles.forEach(function (bundleName) {
            mergeFlags(resultConfig, resultConfig.bundles[bundleName], lmdPlugins, false);
        });

        // Set bundle plugin
        if (bundles.length) {
            resultConfig.bundle = true;
        }
    }
}

function addPluginsFromBundles ( resultConfig ) { if ( resultConfig . bundles ) { var bundles = Object . keys ( resultConfig . bundles ) , lmdPlugins = Object . keys ( LMD_PLUGINS ) ; bundles . forEach ( function ( bundleName ) { mergeFlags ( resultConfig , resultConfig . bundles [ bundleName ] , lmdPlugins , false ) ; } ) ; if ( bundles . length ) { resultConfig . bundle = true ; } } }

For now add plugins from build to bundle

@param resultConfig


function (code, options) {
    var exports = [],
        requires = [],
        bind = [],
        extra_exports = options.extra_exports,
        extra_require = options.extra_require,
        extra_bind = options.extra_bind,
        exportCode,
        bindModuleName;

    // add exports to the module end
    // extra_exports = {name: code, name: code}
    if (typeof extra_exports === "object") {
        for (var exportName in extra_exports) {
            exportCode = extra_exports[exportName];
            exports.push('    ' + JSON.stringify(exportName) + ': ' + exportCode);
        }
        code += '\n\n/* added by builder */\nreturn {\n' + exports.join(',\n') + '\n};';
    } else if (extra_exports) {
        // extra_exports = string
        code += '\n\n/* added by builder */\nreturn ' + extra_exports + ';';
    }

    // change context of module (this)
    // and proxy return value
    // return function(){}.call({name: require('name')});
    if (typeof extra_bind === "object") {
        for (var bindName in extra_bind) {
            bindModuleName = extra_bind[bindName];
            bind.push('    ' + JSON.stringify(bindName) + ': require(' + JSON.stringify(bindModuleName) + ')');
        }
        code = '\nreturn function(){\n\n' + code + '\n}.call({\n' + bind.join(',\n') + '\n});';
    } else if (extra_bind) {
        // return function(){}.call(require('name'));
        code = '\nreturn function(){\n\n' + code + '\n}.call(require(' + JSON.stringify(extra_bind) + '));';
    }

    // add require to the module start
    if (typeof extra_require === "object") {
        // extra_require = [name, name, name]
        if (extra_require instanceof Array) {
            for (var i = 0, c = extra_require.length, moduleName; i < c; i++) {
                moduleName = extra_require[i];
                requires.push('require(' + JSON.stringify(moduleName) + ');');
            }
            code = '/* added by builder */\n' + requires.join('\n') + '\n\n' + code;
        } else {
            // extra_require = {name: name, name: name}
            for (var localName in extra_require) {
                moduleName = extra_require[localName];
                requires.push(localName + ' = require(' + JSON.stringify(moduleName) + ')');
            }
            code = '/* added by builder */\nvar ' + requires.join(',\n    ') + ';\n\n' + code;
        }
    } else if (extra_require) {
        // extra_require = string
        code = '/* added by builder */\nrequire(' + JSON.stringify(extra_require) + ');\n\n' + code;
    }

    return '(function (require) { /* wrapped by builder */\n' + code + '\n})';
}

Wrapper for non-lmd modules files

@param {String}               code
@param {Object}               options
@param {Object|String}        options.extra_exports
@param {Object|String|String} options.extra_require
@param {Object|String|String} options.extra_bind

@returns {String} wrapped code


function (code, moduleOptions, moduleType) {
    switch (moduleType) {
        case "3-party":
            // create lmd module from non-lmd module
            code = wrap3partyModule(code, moduleOptions);
            break;

        case "plain":
            // wrap plain module
            code = wrapPlainModule(code);
            break;

        case "amd":
            // AMD RequireJS
            code = wrapAmdModule(code);
            break;

        case "fd":
        case "fe":
            // wipe tail ;
            code = removeTailSemicolons(code);
    }

    return code;
}

function ( code , moduleOptions , moduleType ) { switch ( moduleType ) { case "3-party" : code = wrap3partyModule ( code , moduleOptions ) ; break ; case "plain" : code = wrapPlainModule ( code ) ; break ; case "amd" : code = wrapAmdModule ( code ) ; break ; case "fd" : case "fe" : code = removeTailSemicolons ( code ) ; } return code ; }

Aggregates all module wrappers

@param code
@param moduleOptions
@param moduleType

@return {String}


function getModuleType (code) {
    var ast;

    if (typeof code === "object") {
        ast = code;
    } else {
        try {
            JSON.parse(code);
            return "json";
        } catch (e) {}

        try {
            ast = parser.parse(code);
        } catch (e) {
            return "string";
        }
    }

    // Empty module
    if (ast.length === 2 && !ast[1].length && ast[0] === 'toplevel') return "plain";

    // ["toplevel",[["defun","depA",["require"],[]]]]
    if (ast && ast.length === 2 &&
        ast[1] && ast[1].length === 1 &&
        ast[1][0][0] === "defun"
        ) {
        return "fd";
    }

    // ["toplevel",[["stat",["function",null,["require"],[]]]]]
    if (ast && ast.length === 2 &&
        ast[1] && ast[1].length === 1 &&
        ast[1][0][0] === "stat" &&
        ast[1][0][1] &&
        ast[1][0][1][0] === "function"
        ) {
        return "fe";
    }

    if (ast) {
        var isAmd = ast[1].every(function (ast) {
            return ast[0] === "stat" &&
                ast[1][0] === "call" &&
                ast[1][1][0] === "name" &&
                ast[1][1][1] === "define";
        });

        if (isAmd) {
            return "amd";
        }
    }

    return "plain";
}

function getModuleType ( code ) { var ast ; if ( typeof code === "object" ) { ast = code ; } else { try { JSON . parse ( code ) ; return "json" ; } catch ( e ) { } try { ast = parser . parse ( code ) ; } catch ( e ) { return "string" ; } } if ( ast . length === 2 && ! ast [ 1 ] . length && ast [ 0 ] === 'toplevel' ) return "plain" ; if ( ast && ast . length === 2 && ast [ 1 ] && ast [ 1 ] . length === 1 && ast [ 1 ] [ 0 ] [ 0 ] === "defun" ) { return "fd" ; } if ( ast && ast . length === 2 && ast [ 1 ] && ast [ 1 ] . length === 1 && ast [ 1 ] [ 0 ] [ 0 ] === "stat" && ast [ 1 ] [ 0 ] [ 1 ] && ast [ 1 ] [ 0 ] [ 1 ] [ 0 ] === "function" ) { return "fe" ; } if ( ast ) { var isAmd = ast [ 1 ] . every ( function ( ast ) { return ast [ 0 ] === "stat" && ast [ 1 ] [ 0 ] === "call" && ast [ 1 ] [ 1 ] [ 0 ] === "name" && ast [ 1 ] [ 1 ] [ 1 ] === "define" ; } ) ; if ( isAmd ) { return "amd" ; } } return "plain" ; }

Checks module type

@param {Object|String} code module AST or module Content

@return {String} df|fe|plain|amd


function d3_transform(m) {
  var r0 = [m.a, m.b],
      r1 = [m.c, m.d],
      kx = d3_transformNormalize(r0),
      kz = d3_transformDot(r0, r1),
      ky = d3_transformNormalize(d3_transformCombine(r1, r0, -kz));
  this.translate = [m.e, m.f];
  this.rotate = Math.atan2(m.b, m.a) * d3_transformDegrees;
  this.scale = [kx, ky || 0];
  this.skew = ky ? kz / ky * d3_transformDegrees : 0;
}

function d3_transform ( m ) { var r0 = [ m . a , m . b ] , r1 = [ m . c , m . d ] , kx = d3_transformNormalize ( r0 ) , kz = d3_transformDot ( r0 , r1 ) , ky = d3_transformNormalize ( d3_transformCombine ( r1 , r0 , - kz ) ) ; this . translate = [ m . e , m . f ] ; this . rotate = Math . atan2 ( m . b , m . a ) * d3_transformDegrees ; this . scale = [ kx , ky || 0 ] ; this . skew = ky ? kz / ky * d3_transformDegrees : 0 ; }

Compute x-scale and normalize the first row. Compute shear and make second row orthogonal to first. Compute y-scale and normalize the second row. Finally, compute the rotation.

Compute x - scale and normalize the first row . Compute shear and make second row orthogonal to first . Compute y - scale and normalize the second row . Finally compute the rotation .


function mousewheel() {
    start.apply(this, arguments);
    if (!d3_behavior_zoomZooming) d3_behavior_zoomZooming = d3_behavior_zoomLocation(d3.svg.mouse(d3_behavior_zoomTarget));
    d3_behavior_zoomTo(d3_behavior_zoomDelta() + xyz[2], d3.svg.mouse(d3_behavior_zoomTarget), d3_behavior_zoomZooming);
  }

function mousewheel ( ) { start . apply ( this , arguments ) ; if ( ! d3_behavior_zoomZooming ) d3_behavior_zoomZooming = d3_behavior_zoomLocation ( d3 . svg . mouse ( d3_behavior_zoomTarget ) ) ; d3_behavior_zoomTo ( d3_behavior_zoomDelta ( ) + xyz [ 2 ] , d3 . svg . mouse ( d3_behavior_zoomTarget ) , d3_behavior_zoomZooming ) ; }


function d3_behavior_zoomDelta() {

  // mousewheel events are totally broken!
  // https://bugs.webkit.org/show_bug.cgi?id=40441
  // not only that, but Chrome and Safari differ in re. to acceleration!
  if (!d3_behavior_zoomDiv) {
    d3_behavior_zoomDiv = d3.select("body").append("div")
        .style("visibility", "hidden")
        .style("top", 0)
        .style("height", 0)
        .style("width", 0)
        .style("overflow-y", "scroll")
      .append("div")
        .style("height", "2000px")
      .node().parentNode;
  }

  var e = d3.event, delta;
  try {
    d3_behavior_zoomDiv.scrollTop = 1000;
    d3_behavior_zoomDiv.dispatchEvent(e);
    delta = 1000 - d3_behavior_zoomDiv.scrollTop;
  } catch (error) {
    delta = e.wheelDelta || (-e.detail * 5);
  }

  return delta * .005;
}

function d3_behavior_zoomDelta ( ) { if ( ! d3_behavior_zoomDiv ) { d3_behavior_zoomDiv = d3 . select ( "body" ) . append ( "div" ) . style ( "visibility" , "hidden" ) . style ( "top" , 0 ) . style ( "height" , 0 ) . style ( "width" , 0 ) . style ( "overflow-y" , "scroll" ) . append ( "div" ) . style ( "height" , "2000px" ) . node ( ) . parentNode ; } var e = d3 . event , delta ; try { d3_behavior_zoomDiv . scrollTop = 1000 ; d3_behavior_zoomDiv . dispatchEvent ( e ) ; delta = 1000 - d3_behavior_zoomDiv . scrollTop ; } catch ( error ) { delta = e . wheelDelta || ( - e . detail * 5 ) ; } return delta * .005 ; }

detect the pixels that would be scrolled by this wheel event


function (data) {
    var config;

    // case data is argv string
    if (typeof data === "string") {
        // try to parse new version
        config = parseArgv(data);

        // its new config argv string
        if (Object.keys(config).length) {
            // translate short params to long one
            config.mode = config.mode || config.m;
            config.output = config.output || config.o;
            config.log = config.log || config.l;
            config.config = config.config || config.c;
            config['no-warn'] = config['no-warn'] || config['no-w'];
            config['source-map'] = config['source-map'] || config['sm'];
            config['source-map-root'] = config['source-map-root'] || config['sm-root'];
            config['source-map-www'] = config['source-map-www'] || config['sm-www'];
            config['source-map-inline'] = config['source-map-inline'] || config['sm-inline'];
        } else {
            // an old argv format, split argv and parse manually
            data = data.split(' ');

            // without mode
            if (availableModes.indexOf(data[2]) === -1) {
                config = {
                    mode: 'main',
                    config: data[2],
                    output: data[3]
                };
            } else { // with mode
                config = {
                    mode: data[2],
                    config: data[3],
                    output: data[4]
                };
            }
        }

    // case data is config object
    } else if (typeof config === "object") {
        // use as is
        config = data;

    // case else
    } else {
        // wut?
        throw new Error('Bad config data');
    }

    config.mode = config.mode || 'main';
    config.warn = !config['no-warn'];
    config.sourcemap = config['source-map'] || false;
    config.www_root = config['source-map-root'] || "";
    config.sourcemap_www = config['source-map-www'] || "";
    config.sourcemap_inline = config['source-map-inline'] || false;
    config.log = config.log || false;

    return config;
}

function
(
data
)
{
var
config
;
if
(
typeof
data
===
"string"
)
{
config
=
parseArgv
(
data
)
;
if
(
Object
.
keys
(
config
)
.
length
)
{
config
.
mode
=
config
.
mode
||
config
.
m
;
config
.
output
=
config
.
output
||
config
.
o
;
config
.
log
=
config
.
log
||
config
.
l
;
config
.
config
=
config
.
config
||
config
.
c
;
config
[
'no-warn'
]
=
config
[
'no-warn'
]
||
config
[
'no-w'
]
;
config
[
'source-map'
]
=
config
[
'source-map'
]
||
config
[
'sm'
]
;
config
[
'source-map-root'
]
=
config
[
'source-map-root'
]
||
config
[
'sm-root'
]
;
config
[
'source-map-www'
]
=
config
[
'source-map-www'
]
||
config
[
'sm-www'
]
;
config
[
'source-map-inline'
]
=
config
[
'source-map-inline'
]
||
config
[
'sm-inline'
]
;
}
else
{
data
=
data
.
split
(
' '
)
;
if
(
availableModes
.
indexOf
(
data
[
2
]
)
===
1
)
{
config
=
{
mode
:
'main'
,
config
:
data
[
2
]
,
output
:
data
[
3
]
}
;
}
else
{
config
=
{
mode
:
data
[
2
]
,
config
:
data
[
3
]
,
output
:
data
[
4
]
}
;
}
}
}
else
if
(
typeof
config
===
"object"
)
{
config
=
data
;
}
else
{
throw
new
Error
(
'Bad config data'
)
;
}
config
.
mode
=
config
.
mode
||
'main'
;
config
.
warn
=
!
config
[
'no-warn'
]
;
config
.
sourcemap
=
config
[
'source-map'
]
||
false
;
config
.
www_root
=
config
[
'source-map-root'
]
||
""
;
config
.
sourcemap_www
=
config
[
'source-map-www'
]
||
""
;
config
.
sourcemap_inline
=
config
[
'source-map-inline'
]
||
false
;
config
.
log
=
config
.
log
||
false
;
return
config
;
}

Formats lmd config

@param  {String|Object} data

@return {Object}


function parse (options, document, callback) {
  xml2js.parseString(document, {
    trim: true,
    normalizeTags: true,
    normalize: true
  }, function (err, result) {
    var source = options.source;

    if (!err) {

      // Process the url input, but break if base.input returns false.
      //   In other words, _.find is looking for a non-falsy err.
      // For now, this can only happen if no outputDir is defined,
      //   which is a fatal bad option problem and will happen immediately.
      _.find(
        // if the sitemap is malformed, just blow up
        result.urlset.url,
        function (urlNode) {
          // optionally ignore current urls by sitemap policy
          var url,
              process = !options.sitemapPolicy ||
                !stillCurrent(urlNode, options);

          if (process) {
            // if sitemap is malformed, just blow up
            url = urlm.parse(urlNode.loc[0]);
            if (!base.input(_.extend({}, options, {
                protocol: url.protocol,
                auth: url.auth,
                hostname: url.hostname,
                port: url.port
              }),
              urlNode.loc[0])
            ) {
              source = urlNode.loc[0];
              err = base.generatorError();
            }
          }

          return err;
        }
      );
    }

    callback(common.prependMsgToErr(err, source, true));
  });
}

function parse ( options , document , callback ) { xml2js . parseString ( document , { trim : true , normalizeTags : true , normalize : true } , function ( err , result ) { var source = options . source ; if ( ! err ) { _ . find ( result . urlset . url , function ( urlNode ) { var url , process = ! options . sitemapPolicy || ! stillCurrent ( urlNode , options ) ; if ( process ) { url = urlm . parse ( urlNode . loc [ 0 ] ) ; if ( ! base . input ( _ . extend ( { } , options , { protocol : url . protocol , auth : url . auth , hostname : url . hostname , port : url . port } ) , urlNode . loc [ 0 ] ) ) { source = urlNode . loc [ 0 ] ; err = base . generatorError ( ) ; } } return err ; } ) ; } callback ( common . prependMsgToErr ( err , source , true ) ) ; } ) ; }

Parse a sitemap document
For each qualifying url element in urlset, call base.input
Stops processing if an error occurs.

@param {Object} options - Sitemap options.
@param {String} options.source - The sitemap url or file.
@param {Boolean} options.sitemapPolicy - True if sitemapPolicy should be enforced.
@param {String} document - An xml document string.
@param {Function} callback - The completion callback.

Parse a sitemap document For each qualifying url element in urlset call base . input Stops processing if an error occurs .


function convert(options, buffer, next, callback) {
  var gunzip = path.extname(options.source) === ".gz";

  if (gunzip) {
    zlib.gunzip(buffer, function (err, result) {
      if (err) {
        callback(common.prependMsgToErr(err, options.source, true));
      } else {
        next(options, result && result.toString(), callback);
      }
    });
  }
  else {
    next(options, buffer.toString(), callback);
  }
}

function convert ( options , buffer , next , callback ) { var gunzip = path . extname ( options . source ) === ".gz" ; if ( gunzip ) { zlib . gunzip ( buffer , function ( err , result ) { if ( err ) { callback ( common . prependMsgToErr ( err , options . source , true ) ) ; } else { next ( options , result && result . toString ( ) , callback ) ; } } ) ; } else { next ( options , buffer . toString ( ) , callback ) ; } }

Convert Buffer input, call next or callback with error

@param {Object} options - Sitemap options.
@param {String} options.source - Sitemap URL.
@param {Buffer} buffer - A node buffer.
@param {Function} next - A function to run after optional conversion applied.
@param {Function} callback - A completion callback.

Convert Buffer input call next or callback with error


function getUrl (options, parseFn, callback) {
  request({
    url: options.source,
    encoding: null,
    timeout: options.timeout() // get the default timeout
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, ["text/xml", "application/xml"]);

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      convert(options, body, parseFn, callback);
    }
  });
}

function getUrl ( options , parseFn , callback ) { request ( { url : options . source , encoding : null , timeout : options . timeout ( ) } , function ( err , res , body ) { var error = err || common . checkResponse ( res , [ "text/xml" , "application/xml" ] ) ; if ( error ) { callback ( common . prependMsgToErr ( error , options . source , true ) ) ; } else { convert ( options , body , parseFn , callback ) ; } } ) ; }

Retrieve the sitemap from a url and call to parse it.

@param {Object} options - Sitemap options.
@param {String} options.source - Sitemap URL.
@param {Function} options.timeout - User supplied run timeout.
@param {Function} parseFn - A data parser function.
@param {Function} callback - A completion callback function.

Retrieve the sitemap from a url and call to parse it .


function getFile (options, parseFn, callback) {
  fs.readFile(options.source, function (err, data) {
    if (err) {
      callback(common.prependMsgToErr(err, options.source, true));
    } else {
      convert(options, data, parseFn, callback);
    }
  });
}

function getFile ( options , parseFn , callback ) { fs . readFile ( options . source , function ( err , data ) { if ( err ) { callback ( common . prependMsgToErr ( err , options . source , true ) ) ; } else { convert ( options , data , parseFn , callback ) ; } } ) ; }

Retrieve the sitemap from a file and call to parse it.

@param {Object} options - Sitemap options.
@param {String} options.source - Sitemap file path.
@param {Function} parseFn - A data parser function.
@param {Function} callback - A completion callback.

Retrieve the sitemap from a file and call to parse it .


function (error, message, quoteInput) {
    var result, prepend, empty = "", quote = "'";

    if (error) {
      if (message) {
        prepend = quoteInput ? empty.concat(quote, message, quote) : message;
      }

      // Force Error instance, coerce given error to a string
      error = error instanceof Error ? error : new Error(empty + error);

      // If message supplied, prepend it
      error.message =
        prepend ? empty.concat(prepend, ": ", error.message) : error.message;

      result = error;
    }

    return result;
  }

function
(
error
,
message
,
quoteInput
)
{
var
result
,
prepend
,
empty
=
""
,
quote
=
"'"
;
if
(
error
)
{
if
(
message
)
{
prepend
=
quoteInput
empty
.
concat
(
quote
,
message
,
quote
)
:
message
;
}
error
=
error
instanceof
Error
error
:
new
Error
(
empty
+
error
)
;
error
.
message
=
prepend
empty
.
concat
(
prepend
,
": "
,
error
.
message
)
:
error
.
message
;
result
=
error
;
}
return
result
;
}

Prepend a message to an Error message.

@param {Object} [error] - An instance of Error. If not, converted to Error.
If undefined, then pass through undefined.
@param {String} [message] - The message to prepend.
@param {Boolean} [quoteInput] - True if the message should be quoted.
@returns {Error} The updated Error instance, undefined if no error.


function (res, mediaTypes) {
    var contentTypeOk,
        result = "status: '" + res.statusCode + "', GET failed.";

    mediaTypes = !Array.isArray(mediaTypes) ? [mediaTypes] : mediaTypes;

    if (res.statusCode === 200) {
      // if content-type exists, and media type found then contentTypeOk
      contentTypeOk =
        res.headers["content-type"] &&
        // empty array and none found return true
        !mediaTypes.every(function(mediaType) {
          // flip -1 to 0 and NOT, so that true == NOT found, found stops loop w/false
          return !~res.headers["content-type"].indexOf(mediaType);
        });

      result = contentTypeOk ? "" : "content-type not one of '"+mediaTypes.join(",")+"'";
    }

    return result;
  }

function
(
res
,
mediaTypes
)
{
var
contentTypeOk
,
result
=
"status: '"
+
res
.
statusCode
+
"', GET failed."
;
mediaTypes
=
!
Array
.
isArray
(
mediaTypes
)
[
mediaTypes
]
:
mediaTypes
;
if
(
res
.
statusCode
===
200
)
{
contentTypeOk
=
res
.
headers
[
"content-type"
]
&&
!
mediaTypes
.
every
(
function
(
mediaType
)
{
return
!
~
res
.
headers
[
"content-type"
]
.
indexOf
(
mediaType
)
;
}
)
;
result
=
contentTypeOk
""
:
"content-type not one of '"
+
mediaTypes
.
join
(
","
)
+
"'"
;
}
return
result
;
}

Simple response checker for remote files.
Expected use in robots.txt or sitemap.xml only.

@param {IncomingMessage} res - The IncomingMessage response to check.
@param {Array} mediaTypes - array of acceptable content-type media type strings.
@returns {String} Error message, empty string (falsy) if OK.

Simple response checker for remote files . Expected use in robots . txt or sitemap . xml only .


function nodeCall (nodeFunc /* args... */) {
  var nodeArgs = Array.prototype.slice.call(arguments, 1);

  return new Promise(function (resolve, reject) {
    /**
     * Resolve a node callback
     */
    function nodeResolver (err, value) {
      if (err) {
        reject(err);
      } else {
        resolve(value);
      }
    }

    nodeArgs.push(nodeResolver);
    nodeFunc.apply(nodeFunc, nodeArgs);
  });
}

function nodeCall ( nodeFunc ) { var nodeArgs = Array . prototype . slice . call ( arguments , 1 ) ; return new Promise ( function ( resolve , reject ) { function nodeResolver ( err , value ) { if ( err ) { reject ( err ) ; } else { resolve ( value ) ; } } nodeArgs . push ( nodeResolver ) ; nodeFunc . apply ( nodeFunc , nodeArgs ) ; } ) ; }

Utility to promisify a Node function

@param {Function} nodeFunc - The node-style function to Promisify.


function prepareWrite (outputPath, callback) {
  var path = pathLib.parse(outputPath);
  var dir = pathLib.join(path.root, path.dir);
  mkdirp(dir, callback);
}

function prepareWrite ( outputPath , callback ) { var path = pathLib . parse ( outputPath ) ; var dir = pathLib . join ( path . root , path . dir ) ; mkdirp ( dir , callback ) ; }

Make sure path exists so a sitemap write can succeed.

@param {String} outputPath - File path and name.
@param {Funcion} callback - Completion callback.

Make sure path exists so a sitemap write can succeed .


function parse (options, document, callback) {
  xml2js.parseString(document, {
    trim: true,
    normalizeTags: true,
    normalize: true
  }, function (err, result) {
    var sitemapUrls = [];
    var sitemapIndexOptions = Object.assign({}, options, {
      outputPath: undefined
    });

    if (!err) {
      // Check if we should process each sitemap in the index.
      _.forEach(
        // if the sitemap index is malformed, just blow up
        result.sitemapindex.sitemap,
        function (sitemapNode) {
          // optionally ignore current sitemaps by sitemap policy
          var shouldProcess = !options.sitemapPolicy ||
            !smLib.stillCurrent(sitemapNode, sitemapIndexOptions);

          if (shouldProcess) {
            // if sitemap index is malformed, just blow up
            sitemapUrls.push(sitemapNode.loc[0]);
          }
        }
      );

      // Edge case message for clarity
      if (sitemapUrls.length === 0) {
        console.log("[*] No sitemaps qualified for processing");
      }

      // Get all the sitemaps, parse them, and generate input for each
      Promise.all(sitemapUrls.map(function (sitemapUrl) {
        var sitemapOptions = Object.assign({}, options, {
          source: sitemapUrl,
          input: "sitemap",
          sitemapOutputDir: false,
          __sitemapIndex: sitemapIndexOptions
        });

        console.log("[+] Loading the sitemap: '"+sitemapUrl+"'");

        return nodeCall(smLib.getUrl, sitemapOptions, processSitemap);
      }))
        .then(function () {
          // Ignore the array of results to this function
          callback();
        })
        .catch(callback);
    } else {
      callback(err);
    }
  });
}

Parse a sitemap index document
For each qualifying sitemap element, download, parse the sitemap.
Stops processing if an error occurs.

@param {Object} options - Sitemap options.
@param {Boolean} options.sitemapPolicy - True if sitemap policy should be used.
@param {String} document - xml document string.
@param {Function} callback - A completion callback.

Parse a sitemap index document For each qualifying sitemap element download parse the sitemap . Stops processing if an error occurs .


function (options, listener) {
    var opts = Object.assign({}, base.defaults(defaults), options);

    return base.run(opts, generateInput, listener);
  }

function ( options , listener ) { var opts = Object . assign ( { } , base . defaults ( defaults ) , options ) ; return base . run ( opts , generateInput , listener ) ; }

Generate the input arguments for snapshots from a robots.txt file.
Each input argument generated calls the listener passing the input object.

@param {Object} options - Sitemap index options.
@param {Function} listener - Callback to receive each input.
@returns {Promise} Resolves to undefined on completion.

Generate the input arguments for snapshots from a robots . txt file . Each input argument generated calls the listener passing the input object .


function generateInput (options) {
  return nodeCall(
    fs.readFile,
    options.source
  )
    .catch(function (err) {
      options._abort(err);
    })
    .then(function (data) {
      var error;

      if (data) {
        data.toString().split('\n').every(function (line) {
          var page = line.replace(/^\s+|\s+$/g, "");

          if (!base.input(options, page)) {
            error = common.prependMsgToErr(base.generatorError(), page, true);
            return false;
          }
          return true;
        });

        if (error) {
          console.error(error);
          options._abort(error);
        }
      }

      base.EOI(textfile);
    });
}

function generateInput ( options ) { return nodeCall ( fs . readFile , options . source ) . catch ( function ( err ) { options . _abort ( err ) ; } ) . then ( function ( data ) { var error ; if ( data ) { data . toString ( ) . split ( '\n' ) . \n every ; ( function ( line ) { var page = line . replace ( / ^\s+|\s+$ / g , "" ) ; if ( ! base . input ( options , page ) ) { error = common . prependMsgToErr ( base . generatorError ( ) , page , true ) ; return false ; } return true ; } ) } if ( error ) { console . error ( error ) ; options . _abort ( error ) ; } } ) ; }

Generate the snapshot arguments from a line oriented text file.
Each line contains a single url we need a snapshot for.

@param {Object} options - input generator options object.
@param {String} options.source - path to the local input file.
@returns {Promise} resolves when complete.

Generate the snapshot arguments from a line oriented text file . Each line contains a single url we need a snapshot for .


function generateInput (options) {
  var result = new Promise(function (resolve, reject) {
    var all;

    if (Array.isArray(options.source)) {
      all = options.source.every(function (sourceUrl) {
        var url = urlm.parse(sourceUrl);
        var opts = Object.assign({}, options, {
          protocol: url.protocol,
          auth: url.auth,
          hostname: url.hostname,
          port: url.port
        });

        if (!base.input(opts, sourceUrl)) {
          reject(
            common.prependMsgToErr(base.generatorError(), sourceUrl, true)
          );
          return false;
        }

        return true;
      });

      if (all) {
        resolve();
      }
    } else {
      reject(new Error("options.source must be an array"));
    }
  });

  return result
    .catch(function (error) {
      options._abort(error);
    })
    .then(function () {
      base.EOI(array);
    });
}

function
generateInput
(
options
)
{
var
result
=
new
Promise
(
function
(
resolve
,
reject
)
{
var
all
;
if
(
Array
.
isArray
(
options
.
source
)
)
{
all
=
options
.
source
.
every
(
function
(
sourceUrl
)
{
var
url
=
urlm
.
parse
(
sourceUrl
)
;
var
opts
=
Object
.
assign
(
{
}
,
options
,
{
protocol
:
url
.
protocol
,
auth
:
url
.
auth
,
hostname
:
url
.
hostname
,
port
:
url
.
port
}
)
;
if
(
!
base
.
input
(
opts
,
sourceUrl
)
)
{
reject
(
common
.
prependMsgToErr
(
base
.
generatorError
(
)
,
sourceUrl
,
true
)
)
;
return
false
;
}
return
true
;
}
)
;
if
(
all
)
{
resolve
(
)
;
}
}
else
{
reject
(
new
Error
(
"options.source must be an array"
)
)
;
}
}
)
;
return
result
.
catch
(
function
(
error
)
{
options
.
_abort
(
error
)
;
}
)
.
then
(
function
(
)
{
base
.
EOI
(
array
)
;
}
)
;
}

Generate the snapshot arguments from an array of pages.
Stops processing if one fails.

@param {Object} options - input generator options object.
@param {String} options.source - Array of urls.
@param {Function} options._abort - Called to abort the output watcher.
@returns {Promise} Resolves to undefined on success.

Generate the snapshot arguments from an array of pages . Stops processing if one fails .


function normalize (obj) {
  var result = obj;
  if (typeof obj !== "function") {
    if (typeof obj !== "undefined") {
      if (Object.prototype.toString.call(obj) !== "[object Object]") {
        result = (function (value) { return function () { return value; }; }(obj));
      } else {
        result = (function (o) { return function (key, passthru) {
          if (o[key] === void 0) {
            return o.__default || (passthru ? key : undefined);
          } else {
            return o[key];
          }
        }; }(obj));
      }
    } else {
      result = function (passthru) { return passthru; };
    }
  }
  return result;
}

function
normalize
(
obj
)
{
var
result
=
obj
;
if
(
typeof
obj
!==
"function"
)
{
if
(
typeof
obj
!==
"undefined"
)
{
if
(
Object
.
prototype
.
toString
.
call
(
obj
)
!==
"[object Object]"
)
{
result
=
(
function
(
value
)
{
return
function
(
)
{
return
value
;
}
;
}
(
obj
)
)
;
}
else
{
result
=
(
function
(
o
)
{
return
function
(
key
,
passthru
)
{
if
(
o
[
key
]
===
void
0
)
{
return
o
.
__default
||
(
passthru
key
:
undefined
)
;
}
else
{
return
o
[
key
]
;
}
}
;
}
(
obj
)
)
;
}
}
else
{
result
=
function
(
passthru
)
{
return
passthru
;
}
;
}
}
return
result
;
}

Normalize the given object to a function.

If the input is a function, return it as is.
If the input is undefined, then a passthru function is generated.
If the input is NOT an Object, then a function returns that value.
If the input is an Object, then a function receives a key and
returns the property value if it exists, otherwise return the __default value.
If a passthru is requested, the key is returned when no __default is found.

@param {Any} obj - Some input to be represented by a function.
@returns {Function} A function to represent the given input.

Normalize the given object to a function .


function supplyMissingDefault (options, name) {
  if (options[name]() === void 0) {
    options[name] = _.wrap(options[name], function(func, key) {
      var res = func(key);
      return res === void 0 ? defaults[name] : res;
    });
  }
}

function supplyMissingDefault ( options , name ) { if ( options [ name ] ( ) === void 0 ) { options [ name ] = _ . wrap ( options [ name ] , function ( func , key ) { var res = func ( key ) ; return res === void 0 ? defaults [ name ] : res ; } ) ; } }

If a normalized option was overriden, but a default still undefined, supply one.
The result could still be undefined, but only if the default is undefined.

If calling the normalized option with no args is undefined, supply missing default.

@param {Object} options - Input generator options.
@param {String} name - The name of the property to supply missing default for.

If a normalized option was overriden but a default still undefined supply one . The result could still be undefined but only if the default is undefined .


function prepOptions (options, listener) {
  // Ensure defaults are represented
  common.ensure(options, defaults);

  // Normalize certain arguments if they are not functions.
  // The certain arguments are per-page options.
  // However, outputPath is a special case
  [
    "selector", "timeout", "useJQuery", "verbose", "phantomjsOptions"
  ]
  .forEach(function (perPageOption) {
    options[perPageOption] = normalize(options[perPageOption]);
    supplyMissingDefault(options, perPageOption);
  });

  options._inputEmitter = new EventEmitter();
  options._inputEmitter.on("input", listener);
}

function prepOptions ( options , listener ) { common . ensure ( options , defaults ) ; [ "selector" , "timeout" , "useJQuery" , "verbose" , "phantomjsOptions" ] . forEach ( function ( perPageOption ) { options [ perPageOption ] = normalize ( options [ perPageOption ] ) ; supplyMissingDefault ( options , perPageOption ) ; } ) ; options . _inputEmitter = new EventEmitter ( ) ; options . _inputEmitter . on ( "input" , listener ) ; }

Prepare options for use by an input generator.

@param {Object} options - Input generator options with or without defaults.
@param {Function} listener - The input generator listener.

Prepare options for use by an input generator .


function getOutputPath (options, page, parse) {
  var pagePart = urlm.parse(page),
      // if outputPath was normalized with an object, let the key passthru
      outputPath = options.outputPath(page, true);

  // check for bad output path
  if (!outputPath) {
    return false;
  }

  // if the outputPath is really still a url, fix it to path+hash
  if (common.isUrl(outputPath)) {
    outputPath = pagePart.path + (pagePart.hash ? pagePart.hash : "");
  }

  // if the caller wants the url parse output, return it
  if (parse) {
    parse.url = pagePart;
  }

  return outputPath;
}

function getOutputPath ( options , page , parse ) { var pagePart = urlm . parse ( page ) , outputPath = options . outputPath ( page , true ) ; if ( ! outputPath ) { return false ; } if ( common . isUrl ( outputPath ) ) { outputPath = pagePart . path + ( pagePart . hash ? pagePart . hash : "" ) ; } if ( parse ) { parse . url = pagePart ; } return outputPath ; }

Get an output path for a page.

Parse a given page as a url and return its corresponding outputPath.
If parse is an object, parse.url will be a reference to the url parse.
Returns false if no output path found.

@param {Object} options - output path options.
@param {Function} options.outputPath - Derives the outputPath from page.
@param {String} page - The url to the page resource.
@param {Object} [parse] - Supply to get page (url) parse output in prop `url`.
@returns {String} The 'path' part of the file output path.


function mapOutputFile (options, page, parse) {
  if (!_.isFunction(options.outputPath)) {
    options.outputPath = normalize(options.outputPath);
  }

  var outputPath = getOutputPath(options, page, parse);
  var outputDir = options.outputDir;
  var fileName = "index.html";

  if (options.sitemapOutputDir) {
    outputDir = path.join(options.outputDir, options.sitemapOutputDir);
    fileName = "";
  }

  return ( outputPath && path.join(outputDir, outputPath, fileName) ) || false;
}

function mapOutputFile ( options , page , parse ) { if ( ! _ . isFunction ( options . outputPath ) ) { options . outputPath = normalize ( options . outputPath ) ; } var outputPath = getOutputPath ( options , page , parse ) ; var outputDir = options . outputDir ; var fileName = "index.html" ; if ( options . sitemapOutputDir ) { outputDir = path . join ( options . outputDir , options . sitemapOutputDir ) ; fileName = "" ; } return ( outputPath && path . join ( outputDir , outputPath , fileName ) ) || false ; }

Using the options, map the given page to an output path.

@param {Object} options - The input generator options.
@param {String} options.outputDir - The output directory root path.
@param {String} [options.outputPath] - optional function or object to map pages to outputPaths.
@param {String} [options.sitemapOutputDir] - optional root path for sitemaps within outputDir.
@returns {String|Boolean} The full path to the output file or false on failure.

Using the options map the given page to an output path .


function (options, generator, listener) {
    options = options || {};
    prepOptions(options, listener);
    return generator(options);
  }

function ( options , generator , listener ) { options = options || { } ; prepOptions ( options , listener ) ; return generator ( options ) ; }

Run the input generator.

@param {Object} options - Input generator options.
@param {Function} generator - The input generator to run.
@param {Function} listener - The callback that receives each input generated.
@returns {Promise} The promise returned by the input generator.


function (options, page) {
    var parse = {};
    var outputFile = mapOutputFile(options, page, parse);

    if (outputFile) {

      options._inputEmitter.emit("input", {
        outputFile: outputFile,
        // make the url
        url: urlm.format({
                protocol: options.protocol,
                auth: options.auth,
                hostname: options.hostname,
                port: options.port,
                pathname: parse.url.pathname,
                search: parse.url.search,
                hash: parse.url.hash
              }),
        // map the input page to a selector
        selector: options.selector(page),
        // map the input page to a timeout
        timeout: options.timeout(page),
        checkInterval: options.checkInterval,
        // map the input page to a useJQuery flag
        useJQuery: options.useJQuery(page),
        // map the input page to a verbose flag
        verbose: options.verbose(page),
        // map the input page to phantomJS options
        phantomjsOptions: options.phantomjsOptions(page),
        // useful for testing, debugging
        __page: page
      });

    }

    return outputFile;
  }

function ( options , page ) { var parse = { } ; var outputFile = mapOutputFile ( options , page , parse ) ; if ( outputFile ) { options . _inputEmitter . emit ( "input" , { outputFile : outputFile , url : urlm . format ( { protocol : options . protocol , auth : options . auth , hostname : options . hostname , port : options . port , pathname : parse . url . pathname , search : parse . url . search , hash : parse . url . hash } ) , selector : options . selector ( page ) , timeout : options . timeout ( page ) , checkInterval : options . checkInterval , useJQuery : options . useJQuery ( page ) , verbose : options . verbose ( page ) , phantomjsOptions : options . phantomjsOptions ( page ) , __page : page } ) ; } return outputFile ; }

Generate the input
Emit the event that contains the input hash

@param {Object} options - input generator options.
@param {String} options.protocol - https?
@param {String} options.auth - user:pass for old style urls.
@param {String} options.hostname - The hostname part of the url.
@param {Number} options.port - The port part of the url.
@param {Function} options.selector - Produces a selector given a page url.
@param {Function} options.timeout - Produces a timeout given a page url.
@param {Number} options.checkInterval - The millisecond resolute poll interval.
@param {Function} options.userJQuery - Produces a jquery bool given a page url.
@param {Function} options.verbose - Produces a verbose bool given a page url.
@param {Function} options.phantomjsOptions - Produces phantomjs options given a page url.
@param {Function} options._inputEmitter - The EventEmitter to generate input.
@param {String} page - The url to the page resource.
@returns {String} The full output file path.


function pathExists (path, options) {
  options = options || {
    returnFile: false
  };

  // Defaults to F_OK
  return nodeCall(fs.access, path)
    .then(function () {
      return options.returnFile ? path : true;
    })
    .catch(function () {
      if (fs.existsSync(path)) {
        return options.returnFile ? path : true;
      }
      return false;
    });
}

function pathExists ( path , options ) { options = options || { returnFile : false } ; return nodeCall ( fs . access , path ) . then ( function ( ) { return options . returnFile ? path : true ; } ) . catch ( function ( ) { if ( fs . existsSync ( path ) ) { return options . returnFile ? path : true ; } return false ; } ) ; }

Check if a path exists.

@param {String} path - The full path to the file to check.
@param {Object} [options] - fileExists options.
@param {Boolean} [options.returnFile] - If true, resolve to input filename on
success. Otherwise, resolve to boolean. Defaults to false (boolean).
@returns {Promise} Resolves to true (or file) if exists, false otherwise.


function () {
  // If the path we're given by phantomjs is to a .cmd, it is pointing to a global copy.
  // Using the cmd as the process to execute causes problems cleaning up the processes
  // so we walk from the cmd to the phantomjs.exe and use that instead.

  var phantomSource = require("phantomjs-prebuilt").path;

  if (path.extname(phantomSource).toLowerCase() === ".cmd") {
    return path.join(path.dirname( phantomSource ), "//node_modules//phantomjs-prebuilt//lib//phantom//bin//phantomjs.exe");
  }

  return phantomSource;
}

Determine the default phantomJS module path. This is overridden by the
phatomjs option.

This function, (technique and concerns) originated from karma-phantomjs-launcher:

The MIT License
Copyright (C) 2011-2013 Google, Inc.

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

Determine the default phantomJS module path . This is overridden by the phatomjs option .


function worker (input, options, notifier, qcb) {
  var cp,
      customModule,
      snapshotScript = options.snapshotScript,
      phantomjsOptions = Array.isArray(input.phantomjsOptions) ? input.phantomjsOptions : [input.phantomjsOptions];

  // If the outputFile has NOT already been seen by the notifier, process.
  if (!notifier.known(input.outputFile)) {

    // map snapshotScript object script to a real path
    if (_.isObject(options.snapshotScript)) {
      snapshotScript = path.join(__dirname, phantomDir, options.snapshotScript.script) + ".js";
      customModule = options.snapshotScript.module;
    }

    cp = spawn(
      options.phantomjs,
      phantomjsOptions.concat([
        snapshotScript,
        input.outputFile,
        input.url,
        input.selector,
        input.timeout,
        input.checkInterval,
        input.useJQuery,
        input.verbose,
        customModule
      ]), { cwd: process.cwd(), stdio: "inherit", detached: true }
    );

    cp.on("error", function (e) {
      notifier.remove(input.outputFile);
      notifier.setError(e);
      console.error(e);
      qcb(e);
    });

    cp.on("exit", function (code) {
      qcb(code);
    });

    // start counting
    notifier.add(input.outputFile, input.timeout);
  }
  else {
    // The input.outputFile is being or has been processed this run.
    qcb(0);
  }
}

function worker ( input , options , notifier , qcb ) { var cp , customModule , snapshotScript = options . snapshotScript , phantomjsOptions = Array . isArray ( input . phantomjsOptions ) ? input . phantomjsOptions : [ input . phantomjsOptions ] ; if ( ! notifier . known ( input . outputFile ) ) { if ( _ . isObject ( options . snapshotScript ) ) { snapshotScript = path . join ( __dirname , phantomDir , options . snapshotScript . script ) + ".js" ; customModule = options . snapshotScript . module ; } cp = spawn ( options . phantomjs , phantomjsOptions . concat ( [ snapshotScript , input . outputFile , input . url , input . selector , input . timeout , input . checkInterval , input . useJQuery , input . verbose , customModule ] ) , { cwd : process . cwd ( ) , stdio : "inherit" , detached : true } ) ; cp . on ( "error" , function ( e ) { notifier . remove ( input . outputFile ) ; notifier . setError ( e ) ; console . error ( e ) ; qcb ( e ) ; } ) ; cp . on ( "exit" , function ( code ) { qcb ( code ) ; } ) ; notifier . add ( input . outputFile , input . timeout ) ; } else { qcb ( 0 ) ; } }

The worker task that launches phantomjs.

@param {Object} input - An input object from an input generator.
@param {Array} input.phantomjsOptions - Actual Phantomjs options.
@param {String} input.outputFile - The output file that the html is written to.
@param {String} input.url - The url to get the html from.
@param {String} input.selector - The selector to wait for.
@param {String} input.timeout - The phantomjs timeout.
@param {String} input.checkInterval - The interval to poll for results.
@param {Boolean} input.useJQuery - True if selector is jquery selector.
@param {Boolean} input.verbose - True if verbose output is desired.
@param {Object} options - Input options.
@param {Object} options.snapshotScript - Snapshot script options.
@param {String} options.snapshotScript.script - Snapshot script name.
@param {String} options.snapshotScript.module - Snapshot script module.
@param {String} options.phantomjs - Full path to the Phantomjs exe.
@param {Object} notifier - The async Notifier instance for this run.
@param {Function} notifier.remove - Remove an outputFile from notifier.
@param {Function} notifier.setError - Set an error on the notifier.
@param {Function} notifier.add - Add a file, timeout to the notifier.
@param {Function} notifier.known - Check if a file is or has been processed.
@param {Function} qcb - async.queue callback function.

The worker task that launches phantomjs .


function prepOptions (options) {
  // ensure this module's defaults are represented in the options.
  common.ensure(options, defaults);

  // if array data source, ensure input type is "array".
  if (Array.isArray(options.source)) {
    options.input = "array";
  }
}

function prepOptions ( options ) { common . ensure ( options , defaults ) ; if ( Array . isArray ( options . source ) ) { options . input = "array" ; } }

Prepare html snapshots options.

@param {Object} options - html snapshots options.
@param {Array|String} options.source - html snapshots data source.


function (options, listener) {
    var inputGenerator, notifier, started, result, q, emitter, completion;

    options = options || {};
    prepOptions(options);

    // create the inputGenerator, default to robots
    inputGenerator = inputFactory.create(options.input);

    // clean the snapshot output directory
    if (options.outputDirClean) {
      rimraf(options.outputDir);
    }

    // start async completion notification.
    notifier = new Notifier();
    emitter = new EventEmitter();
    started = notifier.start(options.pollInterval, inputGenerator,
      function (err, completed) {
        emitter.emit("complete", err, completed);
      });

    if (started) {
      // create the completion Promise.
      completion = new Promise(function (resolve, reject) {
        function completionResolver (err, completed) {
          try {
            _.isFunction(listener) && listener(err, completed);
          } catch (e) {
            console.error("User supplied listener exception", e);
          }
          if (err) {
            err.notCompleted = notifier.filesNotDone;
            err.completed = completed;
            reject(err);
          } else {
            resolve(completed);
          }
        }
        emitter.addListener("complete", completionResolver);
      });

      // create a worker queue with a parallel process limit.
      q = asyncLib.queue(function (task, callback) {
        task(_.once(callback));
      }, options.processLimit);

      // have the queue call notifier.empty when last item
      //  from the queue is given to a worker.
      q.empty = notifier.qEmpty.bind(notifier);

      // expose abort callback to input generators via options.
      options._abort = function (err) {
        notifier.abort(q, err);
      };

      // generate input for the snapshots.
      result = inputGenerator.run(options, function (input) {
        // give the worker the input and place into the queue
        q.push(_.partial(worker, input, options, notifier));
      })
        // after input generation, resolve on browser completion.
        .then(function () {
          return completion;
        });
    } else {
      result = Promise.reject("failed to start async notifier");
    }

    return result;
  }

function
(
options
,
listener
)
{
var
inputGenerator
,
notifier
,
started
,
result
,
q
,
emitter
,
completion
;
options
=
options
||
{
}
;
prepOptions
(
options
)
;
inputGenerator
=
inputFactory
.
create
(
options
.
input
)
;
if
(
options
.
outputDirClean
)
{
rimraf
(
options
.
outputDir
)
;
}
notifier
=
new
Notifier
(
)
;
emitter
=
new
EventEmitter
(
)
;
started
=
notifier
.
start
(
options
.
pollInterval
,
inputGenerator
,
function
(
err
,
completed
)
{
emitter
.
emit
(
"complete"
,
err
,
completed
)
;
}
)
;
if
(
started
)
{
completion
=
new
Promise
(
function
(
resolve
,
reject
)
{
function
completionResolver
(
err
,
completed
)
{
try
{
_
.
isFunction
(
listener
)
&&
listener
(
err
,
completed
)
;
}
catch
(
e
)
{
console
.
error
(
"User supplied listener exception"
,
e
)
;
}
if
(
err
)
{
err
.
notCompleted
=
notifier
.
filesNotDone
;
err
.
completed
=
completed
;
reject
(
err
)
;
}
else
{
resolve
(
completed
)
;
}
}
emitter
.
addListener
(
"complete"
,
completionResolver
)
;
}
)
;
q
=
asyncLib
.
queue
(
function
(
task
,
callback
)
{
task
(
_
.
once
(
callback
)
)
;
}
,
options
.
processLimit
)
;
q
.
empty
=
notifier
.
qEmpty
.
bind
(
notifier
)
;
options
.
_abort
=
function
(
err
)
{
notifier
.
abort
(
q
,
err
)
;
}
;
result
=
inputGenerator
.
run
(
options
,
function
(
input
)
{
q
.
push
(
_
.
partial
(
worker
,
input
,
options
,
notifier
)
)
;
}
)
.
then
(
function
(
)
{
return
completion
;
}
)
;
}
else
{
result
=
Promise
.
reject
(
"failed to start async notifier"
)
;
}
return
result
;
}

Run all the snapshots using the requested inputGenerator

@param {Object} options - ALL user supplied html snapshots options.
@param {String} options.outputDir - Directory to write the html files to.
@param {String} [options.input] - Input source type "robots", "sitemap", "array";
Defaults to "robots".
@param {Boolean} [options.outputDirClean] - True if output dir should be rm -rf;
Defaults to false.
@param {Number} [options.pollInterval] -
@param {Function} [listener] - User supplied optional callback.
@returns {Promise} Resolves on completion.


function createLockFactory () {
  // Create the per instance async lock.
  var lock = new AsyncLock();

  // Make a random id
  var rid = crypto.randomBytes(16).toString("hex");

  /**
   * Force a serial execution context.
   *
   * @param {Function} fn - The function to guard.
   * @param {Number} timeout - The max time to wait for the lock.
   */
  return function lockFactory (fn, timeout) {
    return function protectedContext () {
      lock.acquire("cs-guard-" + rid, function (done) {
        fn(function () {
          done(null, 0);
        });
      }, NOOP, {
        timeout: timeout
      });
    };
  };
}

function createLockFactory ( ) { var lock = new AsyncLock ( ) ; var rid = crypto . randomBytes ( 16 ) . toString ( "hex" ) ; return function lockFactory ( fn , timeout ) { return function protectedContext ( ) { lock . acquire ( "cs-guard-" + rid , function ( done ) { fn ( function ( ) { done ( null , 0 ) ; } ) ; } , NOOP , { timeout : timeout } ) ; } ; } ; }


function Notifier () {

  // Create the serial execution context mechanism.
  this.csFactory = createLockFactory();

  // The private files collection
  // Contains a file and timer: "filename": {timer: timerid}
  // Used for tracking work left to do. When empty, work is done.
  this.files = {};

  // Contains files successfully processed
  this.filesDone = [];

  // Contains files unsuccessfully processed
  this.filesNotDone = [];

  // true if a timeout occurred, or set by abort
  this.errors = [];

  // the holder of the current failure timeout padding
  this.padTimeout = TIMEOUT_PAD_FLOOR;

  // our reference to the listener
  this.callback = null;

  // our reference to the watcher (an interval id)
  // initial value undefined is important
  // this.watcher;

  // the working pollInterval for this run
  // this.interval

  // flag set by qEmpty callback
  // when the last item from the queue is given to a worker
  this.qempty = false;
}

function Notifier ( ) { this . csFactory = createLockFactory ( ) ; this . files = { } ; this . filesDone = [ ] ; this . filesNotDone = [ ] ; this . errors = [ ] ; this . padTimeout = TIMEOUT_PAD_FLOOR ; this . callback = null ; this . qempty = false ; }

Notifier Constructor
Polls the outputDir and when all the files exist,
calls the listener indicating the snapshots are done.

Notifier Constructor Polls the outputDir and when all the files exist calls the listener indicating the snapshots are done .


function start (pollInterval, input, listener) {
    var result = (
      pollInterval > 0 &&
      typeof listener === "function" &&
      (!!input)
      );

    if (result) {
      if (this.isStarted()) {
        throw new Error("Notifier already started");
      }

      this.callback = listener;
      this.interval = parseInt(pollInterval, 10);

      // Poll the filesystem for the files to exist
      // Checks the child process expected output to determine success or failure
      // if the file exists, then it succeeded.
      this.watcher = setInterval(this.csFactory(function (done) {
        var self = this;
        var eoi = typeof input.EOI === "function" && input.EOI();

        if (eoi) {
          Promise.all(Object.keys(self.files).map(function (file) {
            return pathExists(file, {
              returnFile: true
            });
          }))
            .then(function (files) {
              var callback = self.callback;

              try {
                files.forEach(function (file) {
                  file && self._remove(file, true);
                });

                if (self._isDone()) {
                  self._closeWatcher();
                  if (self.callback) {
                    self.callback = null;
                    setImmediate(function () {
                      callback(self.getError(), self.filesDone);
                    });
                  }
                }
              } catch (e) {
                console.error(e);
              }
              done();
            });
        } else {
          done();
        }
      }.bind(this), L_WAIT), this.interval);

    } else {
      console.error("Bad poll interval, async listener, or input generator supplied");
    }

    return result;
  }

Start a watch interval, when a file exists, remove it from our files array.
If the files array is empty, call the listener and stop the watch interval.

@param {Number} pollInterval - The millisecond poll interval to check for
file completion.
@param {Object} input - The inputGenerator.
@param {Function} listener - The callback to notify when complete.
@param {Function} input.EOI - The inputGenerator function that signals end
of input.
@returns {Boolean} true if started successfully, false otherwise.

Start a watch interval when a file exists remove it from our files array . If the files array is empty call the listener and stop the watch interval .


function add (outputFile, timeout) {
    var failTimeout = timeout;
    var timer;

    if (!this.isStarted()) {
      throw new Error("MUST call `start` before `add`");
    }

    if (!this._exists(outputFile)) {
      // make sure we evaluate after the child process
      failTimeout = parseInt(timeout, 10) + parseInt(this.padTimeout, 10);
      // Stagger and grow the failure timeout padding, add 1s every 10 processes
      this.padTimeout += 100;

      // setup a timeout handler to detect failure
      timer = setTimeout(this.csFactory(function (done) {
        var self = this;

        // if the output file has not already been removed
        if (self._exists(outputFile)) {
          pathExists(outputFile)
            .then(function (fsExists) {
              var callback = self.callback;

              try {
                if (!fsExists) {
                  self._setError(new Error(
                    "'" + outputFile + "' did not get a snapshot before timeout"
                  ));
                }
                self._remove(outputFile, fsExists);

                if (self._isDone()) {
                  self._closeWatcher();
                  if (self.callback) {
                    self.callback = null;
                    setImmediate(function () {
                      callback(self.getError(), self.filesDone);
                    });
                  }
                }
              } catch (e) {
                console.error(e);
              }
              done();
            });
        } else {
          done();
        }
      }.bind(this), L_WAIT), parseInt(failTimeout, 10));

      // add the file tracking object
      this.files[outputFile] = {
        timer: timer
      };
    }
  }

Add a file to the files array if it's not there.

@param {String} outputFile - The full path to the output file to add.
@param {Number} timeout - timeout milliseconds to wait (max).

Add a file to the files array if it s not there .


function known (outputFile) {
    var result = false;
    this.csFactory(function (done) {
      result =
        this._exists(outputFile) || this.filesDone.indexOf(outputFile) > -1;
      done();
    }.bind(this), L_WAIT)();
    return result;
  }

function known ( outputFile ) { var result = false ; this . csFactory ( function ( done ) { result = this . _exists ( outputFile ) || this . filesDone . indexOf ( outputFile ) > - 1 ; done ( ) ; } . bind ( this ) , L_WAIT ) ( ) ; return result ; }

Check if a file is being processed or has already been processed.

@param {String} outputFile - The outputFile to check.
@returns {Boolean} true if the fi

Check if a file is being processed or has already been processed .


function _remove (outputFile, done) {
    if (this._exists(outputFile)) {
      if (done) {
        this.filesDone.push(outputFile);
      } else {
        this.filesNotDone.push(outputFile);
      }
      clearTimeout(this.files[outputFile].timer);
      delete this.files[outputFile];
    }
  }

function _remove ( outputFile , done ) { if ( this . _exists ( outputFile ) ) { if ( done ) { this . filesDone . push ( outputFile ) ; } else { this . filesNotDone . push ( outputFile ) ; } clearTimeout ( this . files [ outputFile ] . timer ) ; delete this . files [ outputFile ] ; } }

Remove a file from the files array if it's there.
Unprotected version.

@param {String} outputFile - The outputFile to remove.
@param {Boolean} done - If true, add to filesDone collection.

Remove a file from the files array if it s there . Unprotected version .


function remove (outputFile, done) {
    this.csFactory(function (_done) {
      this._remove(outputFile, done);
      _done();
    }.bind(this), L_WAIT)();
  }

function remove ( outputFile , done ) { this . csFactory ( function ( _done ) { this . _remove ( outputFile , done ) ; _done ( ) ; } . bind ( this ) , L_WAIT ) ( ) ; }

Remove a file from the files array if it's there.
Protected version.

@param {String} outputFile - The outputFile to remove.
@param {Boolean} done - If true, add to filesDone collection.

Remove a file from the files array if it s there . Protected version .


function (time) {
          fs.write(options.outputFile, filter(page.content), "w");
          globals.exit(0, "snapshot for "+options.url+" finished in "+time+" ms\n  written to "+options.outputFile);
        }

function
(
time
)
{
fs
.
write
(
options
.
outputFile
,
filter
(
page
.
content
)
,
"w"
)
;
globals
.
exit
(
0
,
"snapshot for "
+
options
.
url
+
" finished in "
+
time
+
" ms\n  written to "
+
\n
)
;
}


function oneline (line, options) {
  var key = "Allow: ",
      index = line.indexOf(key);

  if (index !== -1) {
    var page = line.substr(index + key.length).replace(/^\s+|\s+$/g, "");
    return page.indexOf("*") === -1 && base.input(options, page);
  }

  return true;
}

function
oneline
(
line
,
options
)
{
var
key
=
"Allow: "
,
index
=
line
.
indexOf
(
key
)
;
if
(
index
!==
1
)
{
var
page
=
line
.
substr
(
index
+
key
.
length
)
.
replace
(
/
^\s+|\s+$
/
g
,
""
)
;
return
page
.
indexOf
(
"*"
)
===
1
&&
base
.
input
(
options
,
page
)
;
}
return
true
;
}

Generate input for one line of a simple robots.txt file
Does not support wildcards.

Generate input for one line of a simple robots . txt file Does not support wildcards .


function getRobotsUrl (options, callback) {
  request({
    url: options.source,
    timeout: options.timeout()
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, "text/plain");

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      body.toString().split('\n').every(function(line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          error = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });

      callback(error);
    }
  });
}

function getRobotsUrl ( options , callback ) { request ( { url : options . source , timeout : options . timeout ( ) } , function ( err , res , body ) { var error = err || common . checkResponse ( res , "text/plain" ) ; if ( error ) { callback ( common . prependMsgToErr ( error , options . source , true ) ) ; } else { body . toString ( ) . split ( '\n' ) . \n every ; ( function ( line ) { if ( ! oneline ( line , options ) ) { error = common . prependMsgToErr ( base . generatorError ( ) , line , true ) ; return false ; } return true ; } ) } } ) ; }

Retrieves robots.txt from url and parses it.

@param {Object} options - Robots.txt options
@param {String} options.source - The URL to a robots.txt
@param {Function} options.timeout - Returns the robots.txt request timeout.
@param {Function} callback - A completion callback.

Retrieves robots . txt from url and parses it .


function getRobotsFile (options, callback) {
  fs.readFile(options.source, function (err, data) {
    if (!err) {
      data.toString().split('\n').every(function (line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          err = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });
    }

    callback(err);
  });
}

function getRobotsFile ( options , callback ) { fs . readFile ( options . source , function ( err , data ) { if ( ! err ) { data . toString ( ) . split ( '\n' ) . \n every ; } ( function ( line ) { if ( ! oneline ( line , options ) ) { err = common . prependMsgToErr ( base . generatorError ( ) , line , true ) ; return false ; } return true ; } ) } ) ; }

Reads the robots.txt file and parses it.

Reads the robots . txt file and parses it .


function bubble(values) {

  return values.map(d => {

    if (d.key && d.values) {
      if (d.values[0].key === "undefined") return d.values[0].values[0];
      else d.values = bubble(d.values);
    }

    return d;

  });

}

function bubble ( values ) { return values . map ( d => { if ( d . key && d . values ) { if ( d . values [ 0 ] . key === "undefined" ) return d . values [ 0 ] . values [ 0 ] ; else d . values = bubble ( d . values ) ; } return d ; } ) ; }

Bubbles up values that do not nest to the furthest key.
@param {Array} *values* The "values" of a nest object.
@private

Bubbles up values that do not nest to the furthest key .


function exclude(a, b, v) {
  const aStart = a.start({type: "bigInteger"});
  const bStart = b.start({type: "bigInteger"});
  const aEnd = a.end({type: "bigInteger"});
  const bEnd = b.end({type: "bigInteger"});
  const parts = [];

  // compareTo returns negative if left is less than right

  //       aaa
  //   bbb
  //   aaa
  //       bbb
  if (aStart.compareTo(bEnd) > 0 || aEnd.compareTo(bStart) < 0) {
    return [a.cidr];
  }

  //   aaa
  //   bbb
  if (aStart.compareTo(bStart) === 0 && aEnd.compareTo(bEnd) === 0) {
    return [];
  }

  //   aa
  //  bbbb
  if (aStart.compareTo(bStart) > 0 && aEnd.compareTo(bEnd) < 0) {
    return [];
  }

  // aaaa
  //   bbbb
  // aaaa
  //   bb
  if (aStart.compareTo(bStart) < 0 && aEnd.compareTo(bEnd) <= 0) {
    parts.push({
      start: aStart,
      end: bStart.subtract(one),
    });
  }

  //    aaa
  //   bbb
  //   aaaa
  //   bbb
  if (aStart.compareTo(bStart) >= 0 && aEnd.compareTo(bEnd) > 0) {
    parts.push({
      start: bEnd.add(one),
      end: aEnd,
    });
  }

  //  aaaa
  //   bb
  if (aStart.compareTo(bStart) < 0 && aEnd.compareTo(bEnd) > 0) {
    parts.push({
      start: aStart,
      end: bStart.subtract(one),
    });
    parts.push({
      start: bEnd.add(one),
      end: aEnd,
    });
  }

  const remaining = [];
  for (const part of parts) {
    for (const subpart of subparts(part, v)) {
      remaining.push(formatPart(subpart, v));
    }
  }

  return cidrTools.merge(remaining);
}

function exclude ( a , b , v ) { const aStart = a . start ( { type : "bigInteger" } ) ; const bStart = b . start ( { type : "bigInteger" } ) ; const aEnd = a . end ( { type : "bigInteger" } ) ; const bEnd = b . end ( { type : "bigInteger" } ) ; const parts = [ ] ; if ( aStart . compareTo ( bEnd ) > 0 || aEnd . compareTo ( bStart ) < 0 ) { return [ a . cidr ] ; } if ( aStart . compareTo ( bStart ) === 0 && aEnd . compareTo ( bEnd ) === 0 ) { return [ ] ; } if ( aStart . compareTo ( bStart ) > 0 && aEnd . compareTo ( bEnd ) < 0 ) { return [ ] ; } if ( aStart . compareTo ( bStart ) < 0 && aEnd . compareTo ( bEnd ) <= 0 ) { parts . push ( { start : aStart , end : bStart . subtract ( one ) , } ) ; } if ( aStart . compareTo ( bStart ) >= 0 && aEnd . compareTo ( bEnd ) > 0 ) { parts . push ( { start : bEnd . add ( one ) , end : aEnd , } ) ; } if ( aStart . compareTo ( bStart ) < 0 && aEnd . compareTo ( bEnd ) > 0 ) { parts . push ( { start : aStart , end : bStart . subtract ( one ) , } ) ; parts . push ( { start : bEnd . add ( one ) , end : aEnd , } ) ; } const remaining = [ ] ; for ( const part of parts ) { for ( const subpart of subparts ( part , v ) ) { remaining . push ( formatPart ( subpart , v ) ) ; } } return cidrTools . merge ( remaining ) ; }

exclude b from a and return remainder cidrs


function getMsTimestamp(){
        var ts = new Date().getTime();
        if(lastMsTs >= ts){
            lastMsTs++;
        }
        else{
            lastMsTs = ts;
        }
        return lastMsTs;
    }

function getMsTimestamp ( ) { var ts = new Date ( ) . getTime ( ) ; if ( lastMsTs >= ts ) { lastMsTs ++ ; } else { lastMsTs = ts ; } return lastMsTs ; }


function parseUrl(url) {
        var serverOptions = {
            host: "localhost",
            port: 80
        };
        if(url.indexOf("https") === 0){
            serverOptions.port = 443;
        }
        var host = url.split("://").pop();
        serverOptions.host = host;
        var lastPos = host.indexOf(":");
        if (lastPos > -1) {
            serverOptions.host = host.slice(0,lastPos);
            serverOptions.port = Number(host.slice(lastPos+1,host.length));
        }
        return serverOptions;
    }

function parseUrl ( url ) { var serverOptions = { host : "localhost" , port : 80 } ; if ( url . indexOf ( "https" ) === 0 ) { serverOptions . port = 443 ; } var host = url . split ( "://" ) . pop ( ) ; serverOptions . host = host ; var lastPos = host . indexOf ( ":" ) ; if ( lastPos > - 1 ) { serverOptions . host = host . slice ( 0 , lastPos ) ; serverOptions . port = Number ( host . slice ( lastPos + 1 , host . length ) ) ; } return serverOptions ; }

parsing host and port information from url


function prepareParams(params){
        var str = [];
        for(var i in params){
            str.push(i+"="+encodeURIComponent(params[i]));
        }
        return str.join("&");
    }

function prepareParams ( params ) { var str = [ ] ; for ( var i in params ) { str . push ( i + "=" + encodeURIComponent ( params [ i ] ) ) ; } return str . join ( "&" ) ; }


function stripTrailingSlash(str) {
        if(str.substr(str.length - 1) === "/") {
            return str.substr(0, str.length - 1);
        }
        return str;
    }

function stripTrailingSlash ( str ) { if ( str . substr ( str . length - 1 ) === "/" ) { return str . substr ( 0 , str . length - 1 ) ; } return str ; }


function getProperties(orig, props){
        var ob = {};
        var prop;
        for(var i = 0; i < props.length; i++){
            prop = props[i];
            if(typeof orig[prop] !== "undefined"){
                ob[prop] = orig[prop];
            }
        }
        return ob;
    }

function getProperties ( orig , props ) { var ob = { } ; var prop ; for ( var i = 0 ; i < props . length ; i ++ ) { prop = props [ i ] ; if ( typeof orig [ prop ] !== "undefined" ) { ob [ prop ] = orig [ prop ] ; } } return ob ; }

retrieve only specific properties from object


function add_cly_events(event){
        if(!event.key){
            log("Event must have key property");
            return;
        }
        if(cluster.isMaster){
            if(!event.count){
                event.count = 1;
            }

            var props = ["key", "count", "sum", "dur", "segmentation"];
            var e = getProperties(event, props);
            e.timestamp = getMsTimestamp();
            var date = new Date();
            e.hour = date.getHours();
            e.dow = date.getDay();
            log("Adding event: ", event);
            eventQueue.push(e);
            storeSet("cly_event", eventQueue);
        }
        else{
            process.send({ cly: {event: event} });
        }
    }

function
add_cly_events
(
event
)
{
if
(
!
event
.
key
)
{
log
(
"Event must have key property"
)
;
return
;
}
if
(
cluster
.
isMaster
)
{
if
(
!
event
.
count
)
{
event
.
count
=
1
;
}
var
props
=
[
"key"
,
"count"
,
"sum"
,
"dur"
,
"segmentation"
]
;
var
e
=
getProperties
(
event
,
props
)
;
e
.
timestamp
=
getMsTimestamp
(
)
;
var
date
=
new
Date
(
)
;
e
.
hour
=
date
.
getHours
(
)
;
e
.
dow
=
date
.
getDay
(
)
;
log
(
"Adding event: "
,
event
)
;
eventQueue
.
push
(
e
)
;
storeSet
(
"cly_event"
,
eventQueue
)
;
}
else
{
process
.
send
(
{
cly
:
{
event
:
event
}
}
)
;
}
}

internal method for events, in case there is no consent for custom events, but internal events has consents

internal method for events in case there is no consent for custom events but internal events has consents


function prepareRequest(request) {
        request.app_key = Countly.app_key;
        request.device_id = Countly.device_id;
        request.sdk_name = SDK_NAME;
        request.sdk_version = SDK_VERSION;
        if(Countly.check_consent("location")){
            if(Countly.country_code){
                request.country_code = Countly.country_code;
            }
            if(Countly.city){
                request.city = Countly.city;
            }
            if(Countly.ip_address !== null){
                request.ip_address = Countly.ip_address;
            }
        }
        else{
            request.location = "";
        }

        request.timestamp = getMsTimestamp();
        var date = new Date();
        request.hour = date.getHours();
        request.dow = date.getDay();
    }

function prepareRequest ( request ) { request . app_key = Countly . app_key ; request . device_id = Countly . device_id ; request . sdk_name = SDK_NAME ; request . sdk_version = SDK_VERSION ; if ( Countly . check_consent ( "location" ) ) { if ( Countly . country_code ) { request . country_code = Countly . country_code ; } if ( Countly . city ) { request . city = Countly . city ; } if ( Countly . ip_address !== null ) { request . ip_address = Countly . ip_address ; } } else { request . location = "" ; } request . timestamp = getMsTimestamp ( ) ; var date = new Date ( ) ; request . hour = date . getHours ( ) ; request . dow = date . getDay ( ) ; }

prepare request by adding basic info to it


function toRequestQueue(request){
        if(cluster.isMaster){
            if(!Countly.app_key || !Countly.device_id){
                log("app_key or device_id is missing");
                return;
            }
            prepareRequest(request);

            if(requestQueue.length > queueSize){
                requestQueue.shift();
            }

            requestQueue.push(request);
            storeSet("cly_queue", requestQueue);
        }
        else{
            process.send({ cly: {cly_queue: request} });
        }
    }

function
toRequestQueue
(
request
)
{
if
(
cluster
.
isMaster
)
{
if
(
!
Countly
.
app_key
||
!
Countly
.
device_id
)
{
log
(
"app_key or device_id is missing"
)
;
return
;
}
prepareRequest
(
request
)
;
if
(
requestQueue
.
length
>
queueSize
)
{
requestQueue
.
shift
(
)
;
}
requestQueue
.
push
(
request
)
;
storeSet
(
"cly_queue"
,
requestQueue
)
;
}
else
{
process
.
send
(
{
cly
:
{
cly_queue
:
request
}
}
)
;
}
}


function getMetrics(){
        var m = JSON.parse(JSON.stringify(metrics));

        //getting app version
        m._app_version = m._app_version || Countly.app_version;

        m._os = m._os || os.type();
        m._os_version = m._os_version || os.release();
        platform = os.type();

        log("Got metrics", m);
        return m;
    }

function
getMetrics
(
)
{
var
m
=
JSON
.
parse
(
JSON
.
stringify
(
metrics
)
)
;
m
.
_app_version
=
m
.
_app_version
||
Countly
.
app_version
;
m
.
_os
=
m
.
_os
||
os
.
type
(
)
;
m
.
_os_version
=
m
.
_os_version
||
os
.
release
(
)
;
platform
=
os
.
type
(
)
;
log
(
"Got metrics"
,
m
)
;
return
m
;
}


function makeRequest(url, path, params, callback) {
        try {
            log("Sending HTTP request");
            var serverOptions = parseUrl(url);
            var data = prepareParams(params);
            var method = "GET";
            var options = {
                host: serverOptions.host,
                port: serverOptions.port,
                path: path+"?"+data,
                method: "GET"
            };

            if(data.length >= 2000){
                method = "POST";
            }
            else if(Countly.force_post){
                method = "POST";
            }

            if(method === "POST"){
                options.method = "POST";
                options.path = path;
                options.headers = {
                    "Content-Type": "application/x-www-form-urlencoded",
                    "Content-Length": Buffer.byteLength(data)
                };
            }
            var protocol = http;
            if(url.indexOf("https") === 0){
                protocol = https;
            }
            var req = protocol.request(options, function(res) {
                var str = "";
                res.on("data", function (chunk) {
                    str += chunk;
                });

                res.on("end", function () {
                    if(res.statusCode >= 200 && res.statusCode < 300){
                        callback(false, params, str);
                    }
                    else{
                        callback(true, params);
                    }
                });
            });
            if(method === "POST"){
                // write data to request body
                req.write(data);
            }

            req.on("error", function(err){
                log("Connection failed.", err);
                if (typeof callback === "function") {
                    callback(true, params);
                }
            });

            req.end();
        } catch (e) {
            // fallback
            log("Failed HTTP request", e);
            if (typeof callback === "function") { callback(true, params); }
        }
    }

function
makeRequest
(
url
,
path
,
params
,
callback
)
{
try
{
log
(
"Sending HTTP request"
)
;
var
serverOptions
=
parseUrl
(
url
)
;
var
data
=
prepareParams
(
params
)
;
var
method
=
"GET"
;
var
options
=
{
host
:
serverOptions
.
host
,
port
:
serverOptions
.
port
,
path
:
path
+
"?"
+
data
,
method
:
"GET"
}
;
if
(
data
.
length
>=
2000
)
{
method
=
"POST"
;
}
else
if
(
Countly
.
force_post
)
{
method
=
"POST"
;
}
if
(
method
===
"POST"
)
{
options
.
method
=
"POST"
;
options
.
path
=
path
;
options
.
headers
=
{
"Content-Type"
:
"application/x-www-form-urlencoded"
,
"Content-Length"
:
Buffer
.
byteLength
(
data
)
}
;
}
var
protocol
=
http
;
if
(
url
.
indexOf
(
"https"
)
===
0
)
{
protocol
=
https
;
}
var
req
=
protocol
.
request
(
options
,
function
(
res
)
{
var
str
=
""
;
res
.
on
(
"data"
,
function
(
chunk
)
{
str
+=
chunk
;
}
)
;
res
.
on
(
"end"
,
function
(
)
{
if
(
res
.
statusCode
>=
200
&&
res
.
statusCode
<
300
)
{
callback
(
false
,
params
,
str
)
;
}
else
{
callback
(
true
,
params
)
;
}
}
)
;
}
)
;
if
(
method
===
"POST"
)
{
req
.
write
(
data
)
;
}
req
.
on
(
"error"
,
function
(
err
)
{
log
(
"Connection failed."
,
err
)
;
if
(
typeof
callback
===
"function"
)
{
callback
(
true
,
params
)
;
}
}
)
;
req
.
end
(
)
;
}
catch
(
e
)
{
log
(
"Failed HTTP request"
,
e
)
;
if
(
typeof
callback
===
"function"
)
{
callback
(
true
,
params
)
;
}
}
}


function allSettled(promises) {
    "use strict";
    const wrappedPromises = promises.map((curPromise) => curPromise.reflect());
    return Promise.all(wrappedPromises);
}

function
allSettled
(
promises
)
{
"use strict"
;
const
wrappedPromises
=
promises
.
map
(
(
curPromise
)
=>
curPromise
.
reflect
(
)
)
;
return
Promise
.
all
(
wrappedPromises
)
;
}

Creates a promise that is resolved when all input promises have been
settled.  The returned Promise is resolved with an array of
Promise.Inspection objects.

This is the commonly accepted way of implementing allSettled() in Bluebird.
See:  http://bluebirdjs.com/docs/api/reflect.html

@param promises - The array of input promises.
@returns {Promise<Promise.Inspection[]>} A promise that will be resolved once
all input Promises have settled. The returned Promise will be resolved with a
corresponding array of Promise.Inspection objects.

Creates a promise that is resolved when all input promises have been settled . The returned Promise is resolved with an array of Promise . Inspection objects .


function after(parent, index) {
  var siblings = parent.children
  var sibling = siblings[++index]
  var other

  if (is('WhiteSpaceNode', sibling)) {
    sibling = siblings[++index]

    if (is('PunctuationNode', sibling) && punctuation.test(toString(sibling))) {
      sibling = siblings[++index]
    }

    if (is('WordNode', sibling)) {
      other = sibling
    }
  }

  return other
}

function after ( parent , index ) { var siblings = parent . children var sibling = siblings [ ++ index ] var other if ( is ( 'WhiteSpaceNode' , sibling ) ) { sibling = siblings [ ++ index ] if ( is ( 'PunctuationNode' , sibling ) && punctuation . test ( toString ( sibling ) ) ) { sibling = siblings [ ++ index ] } if ( is ( 'WordNode' , sibling ) ) { other = sibling } } return other }


function classify(value) {
  var type = null
  var normal

  value = value.replace(digits, toWords).split(split, 1)[0]
  normal = lower(value)

  if (requiresA(value)) {
    type = 'a'
  }

  if (requiresAn(value)) {
    type = type === 'a' ? 'a-or-an' : 'an'
  }

  if (!type && normal === value) {
    type = vowel.test(normal.charAt(0)) ? 'an' : 'a'
  }

  return type
}

function classify ( value ) { var type = null var normal value = value . replace ( digits , toWords ) . split ( split , 1 ) [ 0 ] normal = lower ( value ) if ( requiresA ( value ) ) { type = 'a' } if ( requiresAn ( value ) ) { type = type === 'a' ? 'a-or-an' : 'an' } if ( ! type && normal === value ) { type = vowel . test ( normal . charAt ( 0 ) ) ? 'an' : 'a' } return type }