token-translator.js 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263
  1. /**
  2. * @fileoverview Translates tokens between Acorn format and Esprima format.
  3. * @author Nicholas C. Zakas
  4. */
  5. //------------------------------------------------------------------------------
  6. // Requirements
  7. //------------------------------------------------------------------------------
  8. // none!
  9. //------------------------------------------------------------------------------
  10. // Private
  11. //------------------------------------------------------------------------------
  12. // Esprima Token Types
  13. const Token = {
  14. Boolean: "Boolean",
  15. EOF: "<end>",
  16. Identifier: "Identifier",
  17. PrivateIdentifier: "PrivateIdentifier",
  18. Keyword: "Keyword",
  19. Null: "Null",
  20. Numeric: "Numeric",
  21. Punctuator: "Punctuator",
  22. String: "String",
  23. RegularExpression: "RegularExpression",
  24. Template: "Template",
  25. JSXIdentifier: "JSXIdentifier",
  26. JSXText: "JSXText"
  27. };
  28. /**
  29. * Converts part of a template into an Esprima token.
  30. * @param {AcornToken[]} tokens The Acorn tokens representing the template.
  31. * @param {string} code The source code.
  32. * @returns {EsprimaToken} The Esprima equivalent of the template token.
  33. * @private
  34. */
  35. function convertTemplatePart(tokens, code) {
  36. const firstToken = tokens[0],
  37. lastTemplateToken = tokens[tokens.length - 1];
  38. const token = {
  39. type: Token.Template,
  40. value: code.slice(firstToken.start, lastTemplateToken.end)
  41. };
  42. if (firstToken.loc) {
  43. token.loc = {
  44. start: firstToken.loc.start,
  45. end: lastTemplateToken.loc.end
  46. };
  47. }
  48. if (firstToken.range) {
  49. token.start = firstToken.range[0];
  50. token.end = lastTemplateToken.range[1];
  51. token.range = [token.start, token.end];
  52. }
  53. return token;
  54. }
  55. /**
  56. * Contains logic to translate Acorn tokens into Esprima tokens.
  57. * @param {Object} acornTokTypes The Acorn token types.
  58. * @param {string} code The source code Acorn is parsing. This is necessary
  59. * to correct the "value" property of some tokens.
  60. * @constructor
  61. */
  62. function TokenTranslator(acornTokTypes, code) {
  63. // token types
  64. this._acornTokTypes = acornTokTypes;
  65. // token buffer for templates
  66. this._tokens = [];
  67. // track the last curly brace
  68. this._curlyBrace = null;
  69. // the source code
  70. this._code = code;
  71. }
  72. TokenTranslator.prototype = {
  73. constructor: TokenTranslator,
  74. /**
  75. * Translates a single Esprima token to a single Acorn token. This may be
  76. * inaccurate due to how templates are handled differently in Esprima and
  77. * Acorn, but should be accurate for all other tokens.
  78. * @param {AcornToken} token The Acorn token to translate.
  79. * @param {Object} extra Espree extra object.
  80. * @returns {EsprimaToken} The Esprima version of the token.
  81. */
  82. translate(token, extra) {
  83. const type = token.type,
  84. tt = this._acornTokTypes;
  85. if (type === tt.name) {
  86. token.type = Token.Identifier;
  87. // TODO: See if this is an Acorn bug
  88. if (token.value === "static") {
  89. token.type = Token.Keyword;
  90. }
  91. if (extra.ecmaVersion > 5 && (token.value === "yield" || token.value === "let")) {
  92. token.type = Token.Keyword;
  93. }
  94. } else if (type === tt.privateId) {
  95. token.type = Token.PrivateIdentifier;
  96. } else if (type === tt.semi || type === tt.comma ||
  97. type === tt.parenL || type === tt.parenR ||
  98. type === tt.braceL || type === tt.braceR ||
  99. type === tt.dot || type === tt.bracketL ||
  100. type === tt.colon || type === tt.question ||
  101. type === tt.bracketR || type === tt.ellipsis ||
  102. type === tt.arrow || type === tt.jsxTagStart ||
  103. type === tt.incDec || type === tt.starstar ||
  104. type === tt.jsxTagEnd || type === tt.prefix ||
  105. type === tt.questionDot ||
  106. (type.binop && !type.keyword) ||
  107. type.isAssign) {
  108. token.type = Token.Punctuator;
  109. token.value = this._code.slice(token.start, token.end);
  110. } else if (type === tt.jsxName) {
  111. token.type = Token.JSXIdentifier;
  112. } else if (type.label === "jsxText" || type === tt.jsxAttrValueToken) {
  113. token.type = Token.JSXText;
  114. } else if (type.keyword) {
  115. if (type.keyword === "true" || type.keyword === "false") {
  116. token.type = Token.Boolean;
  117. } else if (type.keyword === "null") {
  118. token.type = Token.Null;
  119. } else {
  120. token.type = Token.Keyword;
  121. }
  122. } else if (type === tt.num) {
  123. token.type = Token.Numeric;
  124. token.value = this._code.slice(token.start, token.end);
  125. } else if (type === tt.string) {
  126. if (extra.jsxAttrValueToken) {
  127. extra.jsxAttrValueToken = false;
  128. token.type = Token.JSXText;
  129. } else {
  130. token.type = Token.String;
  131. }
  132. token.value = this._code.slice(token.start, token.end);
  133. } else if (type === tt.regexp) {
  134. token.type = Token.RegularExpression;
  135. const value = token.value;
  136. token.regex = {
  137. flags: value.flags,
  138. pattern: value.pattern
  139. };
  140. token.value = `/${value.pattern}/${value.flags}`;
  141. }
  142. return token;
  143. },
  144. /**
  145. * Function to call during Acorn's onToken handler.
  146. * @param {AcornToken} token The Acorn token.
  147. * @param {Object} extra The Espree extra object.
  148. * @returns {void}
  149. */
  150. onToken(token, extra) {
  151. const tt = this._acornTokTypes,
  152. tokens = extra.tokens,
  153. templateTokens = this._tokens;
  154. /**
  155. * Flushes the buffered template tokens and resets the template
  156. * tracking.
  157. * @returns {void}
  158. * @private
  159. */
  160. const translateTemplateTokens = () => {
  161. tokens.push(convertTemplatePart(this._tokens, this._code));
  162. this._tokens = [];
  163. };
  164. if (token.type === tt.eof) {
  165. // might be one last curlyBrace
  166. if (this._curlyBrace) {
  167. tokens.push(this.translate(this._curlyBrace, extra));
  168. }
  169. return;
  170. }
  171. if (token.type === tt.backQuote) {
  172. // if there's already a curly, it's not part of the template
  173. if (this._curlyBrace) {
  174. tokens.push(this.translate(this._curlyBrace, extra));
  175. this._curlyBrace = null;
  176. }
  177. templateTokens.push(token);
  178. // it's the end
  179. if (templateTokens.length > 1) {
  180. translateTemplateTokens();
  181. }
  182. return;
  183. }
  184. if (token.type === tt.dollarBraceL) {
  185. templateTokens.push(token);
  186. translateTemplateTokens();
  187. return;
  188. }
  189. if (token.type === tt.braceR) {
  190. // if there's already a curly, it's not part of the template
  191. if (this._curlyBrace) {
  192. tokens.push(this.translate(this._curlyBrace, extra));
  193. }
  194. // store new curly for later
  195. this._curlyBrace = token;
  196. return;
  197. }
  198. if (token.type === tt.template || token.type === tt.invalidTemplate) {
  199. if (this._curlyBrace) {
  200. templateTokens.push(this._curlyBrace);
  201. this._curlyBrace = null;
  202. }
  203. templateTokens.push(token);
  204. return;
  205. }
  206. if (this._curlyBrace) {
  207. tokens.push(this.translate(this._curlyBrace, extra));
  208. this._curlyBrace = null;
  209. }
  210. tokens.push(this.translate(token, extra));
  211. }
  212. };
  213. //------------------------------------------------------------------------------
  214. // Public
  215. //------------------------------------------------------------------------------
  216. export default TokenTranslator;