You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
171 lines
4.9 KiB
171 lines
4.9 KiB
'use strict'; |
|
|
|
// token types (note: value shouldn't intersect with used char codes) |
|
var WHITESPACE = 1; |
|
var IDENTIFIER = 2; |
|
var NUMBER = 3; |
|
var STRING = 4; |
|
var COMMENT = 5; |
|
var PUNCTUATOR = 6; |
|
var CDO = 7; |
|
var CDC = 8; |
|
var ATRULE = 14; |
|
var FUNCTION = 15; |
|
var URL = 16; |
|
var RAW = 17; |
|
|
|
var TAB = 9; |
|
var N = 10; |
|
var F = 12; |
|
var R = 13; |
|
var SPACE = 32; |
|
|
|
var TYPE = { |
|
WhiteSpace: WHITESPACE, |
|
Identifier: IDENTIFIER, |
|
Number: NUMBER, |
|
String: STRING, |
|
Comment: COMMENT, |
|
Punctuator: PUNCTUATOR, |
|
CDO: CDO, |
|
CDC: CDC, |
|
Atrule: ATRULE, |
|
Function: FUNCTION, |
|
Url: URL, |
|
Raw: RAW, |
|
|
|
ExclamationMark: 33, // ! |
|
QuotationMark: 34, // " |
|
NumberSign: 35, // # |
|
DollarSign: 36, // $ |
|
PercentSign: 37, // % |
|
Ampersand: 38, // & |
|
Apostrophe: 39, // ' |
|
LeftParenthesis: 40, // ( |
|
RightParenthesis: 41, // ) |
|
Asterisk: 42, // * |
|
PlusSign: 43, // + |
|
Comma: 44, // , |
|
HyphenMinus: 45, // - |
|
FullStop: 46, // . |
|
Solidus: 47, // / |
|
Colon: 58, // : |
|
Semicolon: 59, // ; |
|
LessThanSign: 60, // < |
|
EqualsSign: 61, // = |
|
GreaterThanSign: 62, // > |
|
QuestionMark: 63, // ? |
|
CommercialAt: 64, // @ |
|
LeftSquareBracket: 91, // [ |
|
Backslash: 92, // \ |
|
RightSquareBracket: 93, // ] |
|
CircumflexAccent: 94, // ^ |
|
LowLine: 95, // _ |
|
GraveAccent: 96, // ` |
|
LeftCurlyBracket: 123, // { |
|
VerticalLine: 124, // | |
|
RightCurlyBracket: 125, // } |
|
Tilde: 126 // ~ |
|
}; |
|
|
|
var NAME = Object.keys(TYPE).reduce(function(result, key) { |
|
result[TYPE[key]] = key; |
|
return result; |
|
}, {}); |
|
|
|
// https://drafts.csswg.org/css-syntax/#tokenizer-definitions |
|
// > non-ASCII code point |
|
// > A code point with a value equal to or greater than U+0080 <control> |
|
// > name-start code point |
|
// > A letter, a non-ASCII code point, or U+005F LOW LINE (_). |
|
// > name code point |
|
// > A name-start code point, a digit, or U+002D HYPHEN-MINUS (-) |
|
// That means only ASCII code points has a special meaning and we a maps for 0..127 codes only |
|
var SafeUint32Array = typeof Uint32Array !== 'undefined' ? Uint32Array : Array; // fallback on Array when TypedArray is not supported |
|
var SYMBOL_TYPE = new SafeUint32Array(0x80); |
|
var PUNCTUATION = new SafeUint32Array(0x80); |
|
var STOP_URL_RAW = new SafeUint32Array(0x80); |
|
|
|
for (var i = 0; i < SYMBOL_TYPE.length; i++) { |
|
SYMBOL_TYPE[i] = IDENTIFIER; |
|
} |
|
|
|
// fill categories |
|
[ |
|
TYPE.ExclamationMark, // ! |
|
TYPE.QuotationMark, // " |
|
TYPE.NumberSign, // # |
|
TYPE.DollarSign, // $ |
|
TYPE.PercentSign, // % |
|
TYPE.Ampersand, // & |
|
TYPE.Apostrophe, // ' |
|
TYPE.LeftParenthesis, // ( |
|
TYPE.RightParenthesis, // ) |
|
TYPE.Asterisk, // * |
|
TYPE.PlusSign, // + |
|
TYPE.Comma, // , |
|
TYPE.HyphenMinus, // - |
|
TYPE.FullStop, // . |
|
TYPE.Solidus, // / |
|
TYPE.Colon, // : |
|
TYPE.Semicolon, // ; |
|
TYPE.LessThanSign, // < |
|
TYPE.EqualsSign, // = |
|
TYPE.GreaterThanSign, // > |
|
TYPE.QuestionMark, // ? |
|
TYPE.CommercialAt, // @ |
|
TYPE.LeftSquareBracket, // [ |
|
// TYPE.Backslash, // \ |
|
TYPE.RightSquareBracket, // ] |
|
TYPE.CircumflexAccent, // ^ |
|
// TYPE.LowLine, // _ |
|
TYPE.GraveAccent, // ` |
|
TYPE.LeftCurlyBracket, // { |
|
TYPE.VerticalLine, // | |
|
TYPE.RightCurlyBracket, // } |
|
TYPE.Tilde // ~ |
|
].forEach(function(key) { |
|
SYMBOL_TYPE[Number(key)] = PUNCTUATOR; |
|
PUNCTUATION[Number(key)] = PUNCTUATOR; |
|
}); |
|
|
|
for (var i = 48; i <= 57; i++) { |
|
SYMBOL_TYPE[i] = NUMBER; |
|
} |
|
|
|
SYMBOL_TYPE[SPACE] = WHITESPACE; |
|
SYMBOL_TYPE[TAB] = WHITESPACE; |
|
SYMBOL_TYPE[N] = WHITESPACE; |
|
SYMBOL_TYPE[R] = WHITESPACE; |
|
SYMBOL_TYPE[F] = WHITESPACE; |
|
|
|
SYMBOL_TYPE[TYPE.Apostrophe] = STRING; |
|
SYMBOL_TYPE[TYPE.QuotationMark] = STRING; |
|
|
|
STOP_URL_RAW[SPACE] = 1; |
|
STOP_URL_RAW[TAB] = 1; |
|
STOP_URL_RAW[N] = 1; |
|
STOP_URL_RAW[R] = 1; |
|
STOP_URL_RAW[F] = 1; |
|
STOP_URL_RAW[TYPE.Apostrophe] = 1; |
|
STOP_URL_RAW[TYPE.QuotationMark] = 1; |
|
STOP_URL_RAW[TYPE.LeftParenthesis] = 1; |
|
STOP_URL_RAW[TYPE.RightParenthesis] = 1; |
|
|
|
// whitespace is punctuation ... |
|
PUNCTUATION[SPACE] = PUNCTUATOR; |
|
PUNCTUATION[TAB] = PUNCTUATOR; |
|
PUNCTUATION[N] = PUNCTUATOR; |
|
PUNCTUATION[R] = PUNCTUATOR; |
|
PUNCTUATION[F] = PUNCTUATOR; |
|
// ... hyper minus is not |
|
PUNCTUATION[TYPE.HyphenMinus] = 0; |
|
|
|
module.exports = { |
|
TYPE: TYPE, |
|
NAME: NAME, |
|
|
|
SYMBOL_TYPE: SYMBOL_TYPE, |
|
PUNCTUATION: PUNCTUATION, |
|
STOP_URL_RAW: STOP_URL_RAW |
|
};
|
|
|