diff options
Diffstat (limited to 'trunk/infrastructure/ace/www/lexer_support.js')
| -rw-r--r-- | trunk/infrastructure/ace/www/lexer_support.js | 1068 |
1 files changed, 1068 insertions, 0 deletions
diff --git a/trunk/infrastructure/ace/www/lexer_support.js b/trunk/infrastructure/ace/www/lexer_support.js new file mode 100644 index 0000000..3d54f5c --- /dev/null +++ b/trunk/infrastructure/ace/www/lexer_support.js @@ -0,0 +1,1068 @@ +/** + * Copyright 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* This file is also a Helma module, referenced by its path! */ + +AceLexer = (function lexer_init() { + +// utility functions, make this file self-contained + +function forEach(array, func) { + for(var i=0;i<array.length;i++) { + var result = func(array[i], i); + if (result) break; + } +} + +function map(array, func) { + var result = []; + // must remain compatible with "arguments" pseudo-array + for(var i=0;i<array.length;i++) { + if (func) result.push(func(array[i], i)); + else result.push(array[i]); + } + return result; +} + +function filter(array, func) { + var result = []; + // must remain compatible with "arguments" pseudo-array + for(var i=0;i<array.length;i++) { + if (func(array[i], i)) result.push(array[i]); + } + return result; +} + +function isArray(testObject) { + return testObject && typeof testObject === 'object' && + !(testObject.propertyIsEnumerable('length')) && + typeof testObject.length === 'number'; +} + +// these three lines inspired by Steven Levithan's XRegExp +var singleLineRegex = /(?:[^[\\.]+|\\(?:[\S\s]|$)|\[\^?]?(?:[^\\\]]+|\\(?:[\S\s]|$))*]?)+|\./g; +var backReferenceRegex = /(?:[^\\[]+|\\(?:[^0-9]|$)|\[\^?]?(?:[^\\\]]+|\\(?:[\S\s]|$))*]?)+|\\([0-9]+)/g; +var parenFindingRegex = /(?:[^[(\\]+|\\(?:[\S\s]|$)|\[\^?]?(?:[^\\\]]+|\\(?:[\S\s]|$))*]?|\((?=\?))+|(\()/g; + +// Creates a function that, when called with (string, startIndex), finds the first of the patterns that +// matches the string starting at startIndex (and anchored there). Expects startIndex < string.length. +// The function returns a structure containing "whichCase", a number 0..(patterns.length-1) giving the +// index of the pattern that matched, or -1 if no pattern did, and "result", an array of the kind +// returned by RegExp.exec called on that pattern, or the array that would be returned by matching /[\S\s]/ +// (any single character) if no other pattern matched. Supports the flags 'i', 'm', and 's', where the +// effect of 's' is to make dots match all characters, including newlines. +// Patterns in general are not allowed to match zero-width strings, but a pattern that is specified +// as a regular expression literal with the 'm' flag is considered special, and may be zero-width, +// though as a consequence the match cannot include the final newline of the document. (Other flags +// on regular expression literals are ignored; use the "flags" argument instead.) +function makeRegexSwitch(patterns, flags) { + var numPatterns = patterns.length; + var patternStrings = map(patterns, function (p) { + if ((typeof p) == "string") + return p; // a string + else return p.source; // assume it's a regex + }); + var patternZeros = map(patterns, function (p) { + // using "multiline" is a special way to indicate the reg-ex is zero-width + return ((typeof p) != "string") && p.multiline; + }); + patternStrings.push("[\\S\\s]"); // default case + patternZeros.push(false); + // how many capturing groups each pattern has + var numGroups = map(patternStrings, function (p) { + var count = 0; + p.replace(parenFindingRegex, function (full,paren,offset) { if (paren) count++; }); + return count; + }); + // the group number for each case of the switch + var caseGroupNums = []; + var idx = 1; + forEach(numGroups, function (n) { caseGroupNums.push(idx); idx += n+1; }); + // make a big alternation of capturing groups + var alternation = map(patternStrings, function(p, pi) { + // correct the back-reference numbers + p = p.replace(backReferenceRegex, function (full, num) { + if (num) return "\\"+((+num)+caseGroupNums[pi]); + else return full; + }); + var extra = (patternZeros[pi] ? "[\\S\\s]": ""); // tack on another char for zero-widths + return '('+p+extra+')'; + }).join('|'); + // process regex flags + flags = (flags || ""); + var realFlags = "g"; + for(var i=0;i<flags.length;i++) { + var f = flags.charAt(i); + if (f == "i" || f == "m") realFlags += f; + else if (f == "s") { + alternation = alternation.replace(singleLineRegex, + function (x) { return x==='.' ? "[\\S\\s]" : x; }); + } + } + //console.log(alternation); + var bigRegex = new RegExp(alternation, realFlags); + return function (string, matchIndex) { + bigRegex.lastIndex = matchIndex; + var execResult = bigRegex.exec(string); + var whichCase; + var resultArray = []; + // search linearly for which case matched in the alternation + for(var i=0;i<=numPatterns;i++) { + var groupNum = caseGroupNums[i]; + if (execResult[groupNum]) { + whichCase = i; + for(var j=0;j<=numGroups[i];j++) { + var r = execResult[groupNum+j]; + if (patternZeros[i] && j==0) { + r = r.substring(0, r.length-1); + } + resultArray[j] = r; + } + break; + } + } + if (whichCase == numPatterns) + whichCase = -1; // default case + return {whichCase: whichCase, result: resultArray}; + } +} + + +var tokenClasses = { + 'Token': '', + + 'Text': '', + 'TEST': 'test', + 'Whitespace': 'w', + 'Error': 'err', + 'Other': 'x', + 'Dirty': 'dirty', + + 'Keyword': 'k', + 'Keyword.Constant': 'kc', + 'Keyword.Declaration': 'kd', + 'Keyword.Pseudo': 'kp', + 'Keyword.Reserved': 'kr', + 'Keyword.Type': 'kt', + + 'Name': 'n', + 'Name.Attribute': 'na', + 'Name.Builtin': 'nb', + 'Name.Builtin.Pseudo': 'bp', + 'Name.Class': 'nc', + 'Name.Constant': 'no', + 'Name.Decorator': 'nd', + 'Name.Entity': 'ni', + 'Name.Exception': 'ne', + 'Name.Function': 'nf', + 'Name.Property': 'py', + 'Name.Label': 'nl', + 'Name.Namespace': 'nn', + 'Name.Other': 'nx', + 'Name.Tag': 'nt', + 'Name.Variable': 'nv', + 'Name.Variable.Class': 'vc', + 'Name.Variable.Global': 'vg', + 'Name.Variable.Instance': 'vi', + + 'Literal': 'l', + 'Literal.Date': 'ld', + + 'String': 's', + 'String.Backtick': 'sb', + 'String.Char': 'sc', + 'String.Doc': 'sd', + 'String.Double': 's2', + 'String.Escape': 'se', + 'String.Heredoc': 'sh', + 'String.Interpol': 'si', + 'String.Other': 'sx', + 'String.Regex': 'sr', + 'String.Single': 's1', + 'String.Symbol': 'ss', + + 'Number': 'm', + 'Number.Float': 'mf', + 'Number.Hex': 'mh', + 'Number.Integer': 'mi', + 'Number.Integer.Long': 'il', + 'Number.Oct': 'mo', + + 'Operator': 'o', + 'Operator.Word': 'ow', + + 'Punctuation': 'p', + + 'Comment': 'c', + 'Comment.Multiline': 'cm', + 'Comment.Preproc': 'cp', + 'Comment.Single': 'c1', + 'Comment.Special': 'cs', + + 'Generic': 'g', + 'Generic.Deleted': 'gd', + 'Generic.Emph': 'ge', + 'Generic.Error': 'gr', + 'Generic.Heading': 'gh', + 'Generic.Inserted': 'gi', + 'Generic.Output': 'go', + 'Generic.Prompt': 'gp', + 'Generic.Strong': 'gs', + 'Generic.Subheading': 'gu', + 'Generic.Traceback': 'gt' +} + + +function makeTokenProducer(regexData, flags) { + var data = {}; + var procCasesMap = {}; + + // topological sort of state dependencies + var statesToProcess = []; + var sortedStates = []; + var sortedStatesMap = {}; + for(var state in regexData) statesToProcess.push(state); + while (statesToProcess.length > 0) { + var state = statesToProcess.shift(); + var stateReady = true; + forEach(regexData[state], function (c) { + if ((typeof c) == "object" && c.include) { + var otherState = c.include; + if (/!$/.exec(otherState)) { + otherState = otherState.substring(0, otherState.length-1); + } + if (! sortedStatesMap[otherState]) { + stateReady = false; + return true; + } + } + }); + if (stateReady) { + sortedStates.push(state); + sortedStatesMap[state] = true; + } + else { + // move to end of queue + statesToProcess.push(state); + } + } + + forEach(sortedStates, function(state) { + var cases = regexData[state]; + var procCases = []; + forEach(cases, function (c) { + if ((typeof c) == "object" && c.include) { + var otherState = c.include; + var isBang = false; + if (/!$/.exec(otherState)) { + // "bang" include, returns to other state + otherState = otherState.substring(0, otherState.length-1); + isBang = true; + } + forEach(procCasesMap[otherState], function (d) { + var dd = [d[0], d[1], d[2]]; + if (isBang) { + if (! (d[2] && d[2][0] && d[2][0].indexOf('#pop') != 0)) { + dd[2] = ['#pop', otherState].concat(d[2] || []); + } + } + procCases.push(dd); + }); + } + else procCases.push(c); + }); + procCasesMap[state] = procCases; + data[state] = { + switcher: makeRegexSwitch(map(procCases, function(x) { return x[0]; }), flags), + tokenTypes: map(procCases, function(x) { return x[1]; }), + stateEffects: map(procCases, function(y) { + var x = y[2]; + if (!x) return []; + if (isArray(x)) return x; + return [x]; + }) + } + }); + + // mutates stateStack, calls tokenFunc on each new token in order, returns new index + return function(string, startIndex, stateStack, tokenFunc) { + var stateBefore = stateStack.join('/'); + + while (true) { // loop until non-zero-length token + var stateData = data[stateStack[stateStack.length-1]]; + var switcherResult = stateData.switcher(string, startIndex); + var whichCase = switcherResult.whichCase; + var regexResult = switcherResult.result; + var tokenTypes, stateEffects; + if (whichCase < 0) { + tokenTypes = 'Error'; + stateEffects = null; + } + else { + tokenTypes = stateData.tokenTypes[whichCase]; + stateEffects = stateData.stateEffects[whichCase]; + } + + if (stateEffects) { + forEach(stateEffects, function (se) { + if (se === '#pop') stateStack.pop(); + else if (se === '#popall') { + while (stateStack.length > 0) stateStack.pop(); + } + else stateStack.push(se); + }); + } + var stateAfter = stateStack.join('/'); + + if (regexResult[0].length > 0) { + if ((typeof tokenTypes) === "object" && tokenTypes.bygroups) { + var types = tokenTypes.bygroups; + forEach(types, function (t,i) { + var tkn = { width:regexResult[i+1].length, type:t }; + if (i == 0) tkn.stateBefore = stateBefore; + if (i == (types.length-1)) tkn.stateAfter = stateAfter; + tokenFunc(tkn); + }); + } + else { + tokenFunc({ width:regexResult[0].length, type:tokenTypes, + stateBefore:stateBefore, stateAfter:stateAfter }); + } + return startIndex + regexResult[0].length; + } + } + } +} + +function makeSimpleLexer(tokenProducer) { + function lexString(str, tokenFunc) { + var state = ['root']; + var idx = 0; + while (idx < str.length) { + var i = idx; + idx = tokenProducer(str, idx, state, function (tkn) { + tokenFunc(str.substr(i, tkn.width), tkn.type); + i += tkn.width; + }); + } + } + function lexAsLines(str, tokenFunc, newLineFunc) { + str += "\n"; + var nextNewline = str.indexOf('\n'); + var curIndex = 0; + + lexString(str, function(txt, typ) { + var wid = txt.length; + var widthLeft = wid; + while (widthLeft > 0 && curIndex + wid > nextNewline) { + var w = nextNewline - curIndex; + if (w > 0) { + tokenFunc(str.substr(curIndex, w), typ); + } + curIndex += (w+1); + widthLeft -= (w+1); + if (curIndex < str.length) { + newLineFunc(); + nextNewline = str.indexOf("\n", curIndex); + } + } + if (widthLeft > 0) { + tokenFunc(str.substr(curIndex, widthLeft), typ); + curIndex += widthLeft; + } + }); + } + return {lexString:lexString, lexAsLines:lexAsLines}; +} + +var txtTokenProducer = makeTokenProducer( + { + 'root': [ + [/.*?\n/, 'Text'], + [/.+/, 'Text'] + ] + }, 's'); + +var jsTokenProducer = makeTokenProducer( + { + 'root': [ + [/\/\*[^\w\n]+appjet:version[^\w\n]+[0-9.]+[^\w\n]+\*\/[^\w\n]*(?=\n)/, + 'Comment.Special', 'main'], + [/(?:)/m, 'Text', ['main', 'regex-ready', 'linestart']] + ], + 'whitespace' : [ + [/\n/, 'Text', 'linestart'], + [/[^\S\n]+/, 'Text'], + [/\/\*/, 'Comment', 'longcomment'] + ], + 'common' : [ + {include:'whitespace'}, + [/\"/, 'String.Double', 'dstr'], + [/\'/, 'String.Single', 'sstr'] + ], + 'regex-ready' : [ + {include:'whitespace'}, + [/\/(?:[^[\\\n\/]|\\.|\[\^?]?(?:[^\\\]\n]|\\.)+\]?)+\/[gim]*/, 'String.Regex'], + [/(?:)/m, 'Text', '#pop'] + ], + 'main': [ + [/\"\"\"/, 'String.Doc', 'mstr'], + {include:"common"}, + [/<!--/, 'Comment'], + [/\/\/.*?(?=\n)/, 'Comment'], + [/[\{\}\[\]\(;]/, 'Punctuation', 'regex-ready'], + [/[\).]/, 'Punctuation'], + [/[~\^\*!%&<>\|=:,\/?\\]/, 'Operator', 'regex-ready'], + [/[+-]/, 'Operator'], + ['(import|break|case|catch|const|continue|default|delete|do|else|'+ + 'export|for|function|if|in|instanceof|label|new|return|switch|this|'+ + 'throw|try|typeof|var|void|while|with|abstract|boolean|byte|catch|char|'+ + 'class|const|debugger|double|enum|extends|final|finally|float|goto|implements|'+ + 'int|interface|long|native|package|private|protected|public|short|static|super|'+ + 'synchronized|throws|transient|volatile|let|yield)\\b', 'Keyword'], + [/(true|false|null|NaN|Infinity|undefined)\b/, 'Keyword.Constant'], + [/[$a-zA-Z_][a-zA-Z0-9_]*/, 'Name.Other'], + [/[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, 'Number.Float'], + [/0x[0-9a-f]+/, 'Number.Hex'], + [/[0-9]+/, 'Number.Integer'] + ], + 'csscommon': [ // common outside of style rule brackets + {include:'common'}, + [/\{/, 'Punctuation', 'csscontent'], + [/\:[a-zA-Z0-9_-]+/, 'Name.Decorator'], + [/\.[a-zA-Z0-9_-]+/, 'Name.Class'], + [/\#[a-zA-Z0-9_-]+/, 'Name.Function'], + [/[a-zA-Z0-9_-]+/, 'Name.Tag'], + [/[~\^\*!%&\[\]\(\)<>\|+=@:;,.\/?-]/, 'Operator'] + ], + 'cssmain': [ + [/(@media)([^\S\n]+)(\w+)([^\S\n]*)(\{)/, {bygroups:['Keyword', 'Text', 'String', + 'Text', 'Punctuation']}, 'cssmedia'], + {include:'csscommon'} + ], + 'cssmedia': [ + {include:'csscommon'}, + [/\}/, 'Punctuation', '#pop'] + ], + 'csscontent': [ + {include:'common'}, + [/\}/, 'Punctuation', '#pop'], + [/url\(.*?\)/, 'String.Other'], + ['(azimuth|background-attachment|background-color|'+ + 'background-image|background-position|background-repeat|'+ + 'background|border-bottom-color|border-bottom-style|'+ + 'border-bottom-width|border-left-color|border-left-style|'+ + 'border-left-width|border-right|border-right-color|'+ + 'border-right-style|border-right-width|border-top-color|'+ + 'border-top-style|border-top-width|border-bottom|'+ + 'border-collapse|border-left|border-width|border-color|'+ + 'border-spacing|border-style|border-top|border|caption-side|'+ + 'clear|clip|color|content|counter-increment|counter-reset|'+ + 'cue-after|cue-before|cue|cursor|direction|display|'+ + 'elevation|empty-cells|float|font-family|font-size|'+ + 'font-size-adjust|font-stretch|font-style|font-variant|'+ + 'font-weight|font|height|letter-spacing|line-height|'+ + 'list-style-type|list-style-image|list-style-position|'+ + 'list-style|margin-bottom|margin-left|margin-right|'+ + 'margin-top|margin|marker-offset|marks|max-height|max-width|'+ + 'min-height|min-width|opacity|orphans|outline|outline-color|'+ + 'outline-style|outline-width|overflow|padding-bottom|'+ + 'padding-left|padding-right|padding-top|padding|page|'+ + 'page-break-after|page-break-before|page-break-inside|'+ + 'pause-after|pause-before|pause|pitch|pitch-range|'+ + 'play-during|position|quotes|richness|right|size|'+ + 'speak-header|speak-numeral|speak-punctuation|speak|'+ + 'speech-rate|stress|table-layout|text-align|text-decoration|'+ + 'text-indent|text-shadow|text-transform|top|unicode-bidi|'+ + 'vertical-align|visibility|voice-family|volume|white-space|'+ + 'widows|width|word-spacing|z-index|bottom|left|'+ + 'above|absolute|always|armenian|aural|auto|avoid|baseline|'+ + 'behind|below|bidi-override|blink|block|bold|bolder|both|'+ + 'capitalize|center-left|center-right|center|circle|'+ + 'cjk-ideographic|close-quote|collapse|condensed|continuous|'+ + 'crop|crosshair|cross|cursive|dashed|decimal-leading-zero|'+ + 'decimal|default|digits|disc|dotted|double|e-resize|embed|'+ + 'extra-condensed|extra-expanded|expanded|fantasy|far-left|'+ + 'far-right|faster|fast|fixed|georgian|groove|hebrew|help|'+ + 'hidden|hide|higher|high|hiragana-iroha|hiragana|icon|'+ + 'inherit|inline-table|inline|inset|inside|invert|italic|'+ + 'justify|katakana-iroha|katakana|landscape|larger|large|'+ + 'left-side|leftwards|level|lighter|line-through|list-item|'+ + 'loud|lower-alpha|lower-greek|lower-roman|lowercase|ltr|'+ + 'lower|low|medium|message-box|middle|mix|monospace|'+ + 'n-resize|narrower|ne-resize|no-close-quote|no-open-quote|'+ + 'no-repeat|none|normal|nowrap|nw-resize|oblique|once|'+ + 'open-quote|outset|outside|overline|pointer|portrait|px|'+ + 'relative|repeat-x|repeat-y|repeat|rgb|ridge|right-side|'+ + 'rightwards|s-resize|sans-serif|scroll|se-resize|'+ + 'semi-condensed|semi-expanded|separate|serif|show|silent|'+ + 'slow|slower|small-caps|small-caption|smaller|soft|solid|'+ + 'spell-out|square|static|status-bar|super|sw-resize|'+ + 'table-caption|table-cell|table-column|table-column-group|'+ + 'table-footer-group|table-header-group|table-row|'+ + 'table-row-group|text|text-bottom|text-top|thick|thin|'+ + 'transparent|ultra-condensed|ultra-expanded|underline|'+ + 'upper-alpha|upper-latin|upper-roman|uppercase|url|'+ + 'visible|w-resize|wait|wider|x-fast|x-high|x-large|x-loud|'+ + 'x-low|x-small|x-soft|xx-large|xx-small|yes)\\b', 'Keyword'], + ['(indigo|gold|firebrick|indianred|yellow|darkolivegreen|'+ + 'darkseagreen|mediumvioletred|mediumorchid|chartreuse|'+ + 'mediumslateblue|black|springgreen|crimson|lightsalmon|brown|'+ + 'turquoise|olivedrab|cyan|silver|skyblue|gray|darkturquoise|'+ + 'goldenrod|darkgreen|darkviolet|darkgray|lightpink|teal|'+ + 'darkmagenta|lightgoldenrodyellow|lavender|yellowgreen|thistle|'+ + 'violet|navy|orchid|blue|ghostwhite|honeydew|cornflowerblue|'+ + 'darkblue|darkkhaki|mediumpurple|cornsilk|red|bisque|slategray|'+ + 'darkcyan|khaki|wheat|deepskyblue|darkred|steelblue|aliceblue|'+ + 'gainsboro|mediumturquoise|floralwhite|coral|purple|lightgrey|'+ + 'lightcyan|darksalmon|beige|azure|lightsteelblue|oldlace|'+ + 'greenyellow|royalblue|lightseagreen|mistyrose|sienna|'+ + 'lightcoral|orangered|navajowhite|lime|palegreen|burlywood|'+ + 'seashell|mediumspringgreen|fuchsia|papayawhip|blanchedalmond|'+ + 'peru|aquamarine|white|darkslategray|ivory|dodgerblue|'+ + 'lemonchiffon|chocolate|orange|forestgreen|slateblue|olive|'+ + 'mintcream|antiquewhite|darkorange|cadetblue|moccasin|'+ + 'limegreen|saddlebrown|darkslateblue|lightskyblue|deeppink|'+ + 'plum|aqua|darkgoldenrod|maroon|sandybrown|magenta|tan|'+ + 'rosybrown|pink|lightblue|palevioletred|mediumseagreen|'+ + 'dimgray|powderblue|seagreen|snow|mediumblue|midnightblue|'+ + 'paleturquoise|palegoldenrod|whitesmoke|darkorchid|salmon|'+ + 'lightslategray|lawngreen|lightgreen|tomato|hotpink|'+ + 'lightyellow|lavenderblush|linen|mediumaquamarine|green|'+ + 'blueviolet|peachpuff)\\b', 'Name.Builtin'], + [/\!important/, 'Comment.Preproc'], + [/\#[a-zA-Z0-9]{1,6}/, 'Number'], + [/[\.-]?[0-9]*[\.]?[0-9]+(em|px|\%|pt|pc|in|mm|cm|ex)/, 'Number'], + [/-?[0-9]+/, 'Number'], + [/[~\^\*!%&<>\|+=@:,.\/?-]+/, 'Operator'], + [/[\[\]();]+/, 'Punctuation'], + [/[a-zA-Z][a-zA-Z0-9]+/, 'Name'] + ], + 'linestart': [ + [/\/\*[^\w\n]+appjet:css[^\w\n]+\*\/[^\w\n]*(?=\n)/, 'Comment.Special', + ['#popall', 'root', 'cssmain']], + [/\/\*[^\w\n]+appjet:(\w+)[^\w\n]+\*\/[^\w\n]*(?=\n)/, 'Comment.Special', + ['#popall', 'root', 'main', 'regex-ready']], + [/(?:)/m, 'Text', '#pop'] + ], + 'dstr': [ + [/\"/, 'String.Double', '#pop'], + [/(?=\n)/m, 'String.Double', '#pop'], + [/(\\\\|\\\"|[^\"\n])+/, 'String.Double'] + ], + 'sstr': [ + [/\'/, 'String.Single', '#pop'], + [/(?=\n)/m, 'String.Single', '#pop'], + [/(\\\\|\\\'|[^\'\n])+/, 'String.Single'] + ], + 'longcomment': [ + [/\*\//, 'Comment', '#pop'], + [/\n/, 'Comment'], + [/.+?(?:\n|(?=\*\/))/, 'Comment'] + ], + 'mstr': [ + [/(\\\"\"\"|\n)/, 'String.Doc'], + [/\"\"\"/, 'String.Doc', '#pop'], + [/.+?(?=\\"""|"""|\n)/, 'String.Doc'] + ] + } +); + +function escapeHTML(s) { + var re = /[&<>\'\" ]/g; + if (! re.MAP) { + // persisted across function calls! + re.MAP = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', + ' ': ' ' + }; + } + return s.replace(re, function(c) { return re.MAP[c]; }); +} + +var simpleLexer = makeSimpleLexer(jsTokenProducer); + +function codeStringToHTML(codeString) { + var atLineStart = true; + var html = []; + function tokenFunc(txt, type) { + var cls = tokenClasses[type]; + if (cls) html.push('<tt class="',tokenClasses[type],'">'); + else html.push('<tt>'); + html.push(escapeHTML(txt),'</tt>'); + atLineStart = false; + } + function newLineFunc() { + html.push('<br/>\n'); + atLineStart = true; + } + simpleLexer.lexAsLines(codeString, tokenFunc, newLineFunc); + if (atLineStart) html.push('<br/>\n'); + return html.join(''); +} + +/* ========== Incremental Lexer for ACE ========== */ + +function makeIncrementalLexer(tokenProducer) { + + var tokens = newSkipList(); + var buffer = ""; + var nextId = 1; + var dirtyTokenKeys = []; + var uncoloredRanges = []; + + //top.dbg_uncoloredRanges = function() { return uncoloredRanges; } + //top.dbg_dirtyTokenKeys = function() { return dirtyTokenKeys; } + + function mergeRangesIfTouching(a, b) { + // if a = [a0,a1] and b = [b0,b1] are overlapping or touching, return a single merged range + // else return null + var a0 = a[0], a1 = a[1], b0 = b[0], b1 = b[1]; + if (a1 < b0) return null; + if (b1 < a0) return null; + var c0 = ((a0 < b0) ? a0 : b0); + var c1 = ((a1 > b1) ? a1 : b1); + return [c0,c1]; + } + + function addUncoloredRange(rng) { + // shouldn't this merge existing ranges if the new range overlaps multiple ones? + var done = false; + forEach(uncoloredRanges, function (x, i) { + var merged = mergeRangesIfTouching(x, rng); + if (merged) { + uncoloredRanges[i] = merged; + done = true; + return true; + } + }); + if (! done) { + uncoloredRanges.push(rng); + } + } + + function removeUncoloredRange(rng) { + var i = uncoloredRanges.length-1; + while (i >= 0) { + removeUncoloredRangeFrom(rng, i); + i--; + } + } + + function removeUncoloredRangeFrom(rangeToRemove, containingRangeIndex) { + var idx = containingRangeIndex; + var cont = uncoloredRanges[idx]; + var rem0 = rangeToRemove[0], rem1 = rangeToRemove[1]; + // limit to containing range + if (rem0 < cont[0]) rem0 = cont[0]; + if (rem1 > cont[1]) rem1 = cont[1]; + if (rem1 <= rem0) return; + // splice out uncoloredRanes[containingRangeIndex] for 0, 1, or 2 ranges + uncoloredRanges.splice(idx, 1); + if (cont[0] < rem0) + uncoloredRanges.splice(idx, 0, [cont[0], rem0]); + if (rem1 < cont[1]) + uncoloredRanges.splice(idx, 0, [rem1, cont[1]]); + } + + function prepareTokens(tokenArray) { + forEach(tokenArray, function (t) { + t.key = "$"+(nextId++); + }); + return tokenArray; + } + + function roundBackToTokenBoundary(charOffset) { + return tokens.indexOfOffset(charOffset); + } + function roundForwardToTokenBoundary(charOffset) { + var tokenEnd; + if (charOffset == tokens.totalWidth()) + tokenEnd = tokens.length(); + else { + var endToken = tokens.keyAtOffset(charOffset); + tokenEnd = tokens.indexOfKey(endToken); + // round up to nearest token boundary + if (charOffset > tokens.offsetOfKey(endToken)) { + tokenEnd++; + } + } + return tokenEnd; + } + + // findLexingStartPoint and findLexingEndPoint take a character boundary + // (0 .. buffer.length) and return a token boundary (0 .. tokens.length()) + // that, if not at the document edge, is such that the next token outside + // the boundary has a pre/post lexing state associated with it (i.e is not + // a dirty-region token or in the middle of a multi-token lexing rule). + + function findLexingStartPoint(startChar) { + if (tokens.length() == 0) return 0; + var tokenStart = roundBackToTokenBoundary(startChar); + // expand to not break up a series of tokens from the same + // lexing rule, and to include dirty regions + if (tokenStart > 0) { + var tokenBefore = tokens.atIndex(tokenStart - 1); + while (tokenBefore && (! tokenBefore.stateAfter)) { + tokenStart--; + tokenBefore = tokens.prev(tokenBefore); + } + } + return tokenStart; + } + + function findLexingEndPoint(endChar) { + if (tokens.length() == 0) return 0; + var tokenEnd = roundForwardToTokenBoundary(endChar); + // expand to not break up a series of tokens from the same + // lexing rule, and to include dirty regions + if (tokenEnd < tokens.length()) { + var tokenAfter = tokens.atIndex(tokenEnd); + while (tokenAfter && (! tokenAfter.stateBefore)) { + tokenEnd++; + tokenAfter = tokens.next(tokenAfter); + } + } + return tokenEnd; + } + + function updateBuffer(newBuffer, spliceStart, charsRemoved, charsAdded) { + buffer = newBuffer; + + // back up to new line + if (spliceStart > 0) { + var newStart = buffer.lastIndexOf('\n', spliceStart-1) + 1; + var charsBack = spliceStart - newStart; + spliceStart -= charsBack; + charsRemoved += charsBack; + charsAdded += charsBack; + } + // expand to lexing points + var tokenRangeStart = findLexingStartPoint(spliceStart); + var tokenRangeEnd = findLexingEndPoint(spliceStart + charsRemoved); + + var dirtyWidth = 0; + // make sure to mark at least one token dirty so that deletions correctly cause + // rehighlighting; in practice doesn't come up often except when an entire line + // is cleanly deleted, like deleting a blank line (which doesn't usually affect highlighting) + while (dirtyWidth == 0) { + var curStart = tokens.offsetOfIndex(tokenRangeStart); + var curEnd = tokens.offsetOfIndex(tokenRangeEnd); + dirtyWidth = (curEnd - curStart) + (charsAdded - charsRemoved); + if (dirtyWidth == 0) { + if (curEnd >= tokens.totalWidth()) break; + tokenRangeEnd = findLexingEndPoint(curEnd+1); + } + } + + var dirtyTokens = []; // 0 or 1 of them + if (dirtyWidth > 0) { + dirtyTokens.push({ width: dirtyWidth, type: 'Dirty' }); + } + + //console.log("%d, %d, %d, %d", charsRemoved, charsAdded, + //(curEnd - curStart), dirtyWidth); + + tokens.splice(tokenRangeStart, tokenRangeEnd - tokenRangeStart, + prepareTokens(dirtyTokens)); + + if (tokens.totalWidth() != buffer.length) { + console.error("updateBuffer: Bad total token width: "+ + tokens.totalWidth()+" not "+buffer.length); + } + + forEach(dirtyTokens, function (t) { dirtyTokenKeys.push(t.key); }); + dirtyTokenKeys = filter(dirtyTokenKeys, function (k) { return tokens.containsKey(k); }); + //console.log("after update: %s", toSource(tokens.slice())); + + function applySpliceToIndex(i) { + if (i <= spliceStart) return i; + if (i >= (spliceStart + charsRemoved)) return i + charsAdded - charsRemoved; + return spliceStart; + } + for(var i=uncoloredRanges.length-1; i>=0; i--) { + var r = uncoloredRanges[i]; + r[0] = applySpliceToIndex(r[0]); + r[1] = applySpliceToIndex(r[1]); + if (r[1] <= r[0]) uncoloredRanges.splice(i, 1); + } + } + + function processDirtyToken(dirtyToken, isTimeUp, stopAtChar) { + + //console.time("lexing"); + //var p = PROFILER("lex", false); + var stateStack; + if (! tokens.prev(dirtyToken)) stateStack = ['root']; + else stateStack = tokens.prev(dirtyToken).stateAfter.split('/'); + var newTokens = []; + var dirtyTokenIndex = tokens.indexOfEntry(dirtyToken); + var tokenCount = 0; + var startTime = (new Date()).getTime(); + var stopBasedOnChar = (typeof(stopAtChar) == "number"); + //p.mark("tokenize"); + + var curOffset = tokens.offsetOfEntry(dirtyToken); + var startedOffset = curOffset; + var oldToken = dirtyToken; + var oldTokenOffset = curOffset; + var done = false; + + while ((! done) && (! isTimeUp()) && (! (stopBasedOnChar && curOffset >= stopAtChar))) { + curOffset = tokenProducer(buffer, curOffset, stateStack, + function (t) { newTokens.push(t); }); + while (oldToken && (oldTokenOffset + oldToken.width <= curOffset)) { + oldTokenOffset += oldToken.width; + oldToken = tokens.next(oldToken); + } + if (curOffset == tokens.totalWidth()) { + // hit the end + done = true; + } + else if (oldTokenOffset == curOffset) { + // at a token boundary, the beginning of oldTokenOffset + if (stateStack.join('/') === oldToken.stateBefore) { + // state matches up, we can stop + done = true; + } + } + } + + var endedOffset = curOffset; + var dist = endedOffset - startedOffset; + var tokensToRemove; + var newDirtyToken; + if (dist < dirtyToken.width) { + tokens.setEntryWidth(dirtyToken, dirtyToken.width - dist); + tokensToRemove = 0; + newDirtyToken = dirtyToken; + } + else { + var nextLexingPoint = findLexingEndPoint(endedOffset); + var lexingPointChar = tokens.offsetOfIndex(nextLexingPoint); + if (lexingPointChar == endedOffset && (! done) && endedOffset < tokens.totalWidth()) { + // happened to stop at token boundary before end, but not done lexing, + // so make next token dirty + nextLexingPoint = findLexingEndPoint(endedOffset+1); + lexingPointChar = tokens.offsetOfIndex(nextLexingPoint); + } + var dirtyCharsLeft = lexingPointChar - endedOffset; + if (dirtyCharsLeft > 0) { + newDirtyToken = { width: dirtyCharsLeft, type: 'Dirty' }; + newTokens.push(newDirtyToken); + } + tokensToRemove = nextLexingPoint - dirtyTokenIndex; + } + + //p.mark("prepare"); + prepareTokens(newTokens); + //p.mark("remove"); + tokens.splice(dirtyTokenIndex, tokensToRemove, []); + //p.mark("insert"); + tokens.splice(dirtyTokenIndex, 0, newTokens); + if (tokens.totalWidth() != buffer.length) + console.error("processDirtyToken: Bad total token width: "+ + tokens.totalWidth()+" not "+buffer.length); + //p.end(); + + addUncoloredRange([startedOffset, endedOffset]); + + //console.log("processed chars %d to %d", startedOffset, endedOffset); + //console.timeEnd("lexing"); + + return (newDirtyToken && newDirtyToken.key); + } + + function lexSomeDirty(filter, isTimeUp) { + var newDirtyTokenKeys = []; + + forEach(dirtyTokenKeys, function (dirtyKey) { + if (! tokens.containsKey(dirtyKey)) return; + var dirtyToken = tokens.atKey(dirtyKey); + var filterResult; + if ((! isTimeUp()) && ((filterResult = filter(dirtyToken)))) { + var stopAtChar; + if ((typeof filterResult) == "object" && (typeof filterResult.stopAtChar) == "number") { + stopAtChar = filterResult.stopAtChar; + } + var tkn = processDirtyToken(dirtyToken, isTimeUp, filterResult.stopAtChar); + if (tkn) newDirtyTokenKeys.push(tkn); + } + else { + // leave the token behind + newDirtyTokenKeys.push(dirtyKey); + } + + if (tokens.totalWidth() != buffer.length) + console.error("Bad total token width: "+tokens.totalWidth()+" not "+buffer.length); + + }); + + dirtyTokenKeys = newDirtyTokenKeys; + } + + function lexCharRange(charRange, isTimeUp) { + //var startTime = (new Date()).getTime(); + //function isTimeUp() { return ((new Date()).getTime() - startTime) > timeLimit; } + + if (isTimeUp()) return; + + lexSomeDirty(function (dirtyToken) { + var start = tokens.offsetOfEntry(dirtyToken); + var end = start + dirtyToken.width; + if (end <= charRange[0]) return false; + if (start >= charRange[1]) return false; + //console.log("tokenStart: %d, tokenEnd: %d, visStart: %d, visEnd: %d", + //start, end, charRange[0], charRange[1]); + var result = {}; + if (charRange[1] < end) { + result.stopAtChar = charRange[1]; + } + return result; + }, isTimeUp); + + //if (isTimeUp()) return; + + /* + // highlight the visible area + var i = uncoloredRanges.length-1; + // iterate backwards because we change the array + while (i >= 0) { + var rng = uncoloredRanges[i]; + var start = rng[0], end = rng[1]; + if (start < viewRange[0]) start = viewRange[0]; + if (end > viewRange[1]) end = viewRange[1]; + if (end > start) { + var charsRecolored = recolorFunc(start, end-start, + isTimeUp, getSpansForRange); + removeUncoloredSubrange([start, start+charsRecolored], i); + } + if (isTimeUp()) break; + i--; + }*/ + } + + function tokenToString(tkn) { + return toSource({width:tkn.width, type:tkn.type, stateBefore:tkn.stateBefore, stateAfter:tkn.stateAfter}); + } + + // Calls func(startChar, endChar) on each range of characters that needs to be colored + // in the DOM, based on calls to getSpansForRange (which removes chars from consideration) + // and lexCharRange (which calculates new colors and adds chars for consideration). + // There are usually relatively few uncolored ranges, each of which may be many lines, + // even the whole document. + // func must return true iff any tokens are accessed through getSpansForRange during + // the call. func should not do new lexing. + function forEachUncoloredRange(func, isTimeUp) { + var i = 0; + // uncoloredRanges will change during this function! + // Terminates is time runs out, whole document is colored, + // or the func "passes" on all ranges by returning false. + while (i < uncoloredRanges.length && ! isTimeUp()) { + var rng = uncoloredRanges[i]; + var returnVal = func(rng[0], rng[1], isTimeUp); + if (returnVal) { + // func did something, uncolored ranges may have changed around + i = 0; + } + else { + i++; + } + } + } + + // Like forEachUncoloredRange, but "cropped" to the char range given. For example, + // if no "uncolored ranges" extend by a non-zero amount into the char range, + // func will never be called. + function forEachUncoloredSubrange(startChar, endChar, func, isTimeUp) { + forEachUncoloredRange(function (s, e, isTimeUp2) { + if (s < startChar) s = startChar; + if (e > endChar) e = endChar; + if (e > s) { + return func(s, e, isTimeUp2); + } + return false; + }, isTimeUp); + } + + // This function takes note of what it's passed, and assumes that part of the + // DOM has been taken care of (unless justPeek). + // The "func" takes arguments tokenWidth and tokenClass, and is called on each + // token in the range, with the widths adding up to the range size. + function getSpansForRange(startChar, endChar, func, justPeek) { + + if (startChar == endChar) return; + + var startToken = tokens.atOffset(startChar); + var startTokenStart = tokens.offsetOfEntry(startToken); + var curOffset = startChar; + var curToken = startToken; + while (curOffset < endChar) { + var spanEnd; + if (curToken === startToken) { + spanEnd = startTokenStart + startToken.width; + } + else { + spanEnd = curOffset + curToken.width; + } + if (spanEnd > endChar) spanEnd = endChar; + if (spanEnd > curOffset) { + func(spanEnd - curOffset, tokenClasses[curToken.type]); + } + curOffset = spanEnd; + curToken = tokens.next(curToken); + } + + if (! justPeek) removeUncoloredRange([startChar, endChar]); + } + + function markRangeUncolored(start, end) { + addUncoloredRange([start, end]); + } + + return { + updateBuffer: updateBuffer, + lexCharRange: lexCharRange, + getSpansForRange: getSpansForRange, + forEachUncoloredSubrange: forEachUncoloredSubrange, + markRangeUncolored: markRangeUncolored + }; +} + +/* ========== End Incremental Lexer ========== */ + +tokenProds = {js: jsTokenProducer, txt: txtTokenProducer}; + +function getTokenProducer(type) { + return tokenProds[type || 'txt'] || tokenProds['txt']; +} + +function getIncrementalLexer(type) { + return makeIncrementalLexer(getTokenProducer(type)); +} +function getSimpleLexer(type) { + return makeSimpleLexer(getTokenProducer(type)); +} + +return {getIncrementalLexer:getIncrementalLexer, getSimpleLexer:getSimpleLexer, + codeStringToHTML:codeStringToHTML}; + +})(); |