Tokenizer = function(rules, flag) {
    flag = flag ? "g" + flag : "g";
    this.rules = rules;

    this.regExps = {};
    this.matchMappings = {};
    for ( var key in this.rules) {
        var rule = this.rules[key];
        var state = rule;
        var ruleRegExps = [];
        var matchTotal = 0;
        var mapping = this.matchMappings[key] = {};

        for ( var i = 0; i < state.length; i++) {

            if (state[i].regex instanceof RegExp)
                state[i].regex = state[i].regex.toString().slice(1, -1);

            // Count number of matching groups. 2 extra groups from the full match
            // And the catch-all on the end (used to force a match);
            var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;

            // Replace any backreferences and offset appropriately.
            var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
                return "\\" + (parseInt(digit, 10) + matchTotal + 1);
            });

            if (matchcount > 1 && state[i].token.length !== matchcount-1)
                throw new Error("Matching groups and length of the token array don't match in rule #" + i + " of state " + key);

            mapping[matchTotal] = {
                rule: i,
                len: matchcount
            };
            matchTotal += matchcount;

            ruleRegExps.push(adjustedregex);
        }

        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
    }
}
new Tokenizer(rules, flag)
- rules (Object): The highlighting rules
- flag (String): Any additional regular expression flags to pass (like "i" for case insensitive)

Constructs a new tokenizer based on the given rules and flags.