scss-tokenize.js 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335
  1. 'use strict'
  2. const SINGLE_QUOTE = "'".charCodeAt(0)
  3. const DOUBLE_QUOTE = '"'.charCodeAt(0)
  4. const BACKSLASH = '\\'.charCodeAt(0)
  5. const SLASH = '/'.charCodeAt(0)
  6. const NEWLINE = '\n'.charCodeAt(0)
  7. const SPACE = ' '.charCodeAt(0)
  8. const FEED = '\f'.charCodeAt(0)
  9. const TAB = '\t'.charCodeAt(0)
  10. const CR = '\r'.charCodeAt(0)
  11. const OPEN_SQUARE = '['.charCodeAt(0)
  12. const CLOSE_SQUARE = ']'.charCodeAt(0)
  13. const OPEN_PARENTHESES = '('.charCodeAt(0)
  14. const CLOSE_PARENTHESES = ')'.charCodeAt(0)
  15. const OPEN_CURLY = '{'.charCodeAt(0)
  16. const CLOSE_CURLY = '}'.charCodeAt(0)
  17. const SEMICOLON = ';'.charCodeAt(0)
  18. const ASTERISK = '*'.charCodeAt(0)
  19. const COLON = ':'.charCodeAt(0)
  20. const AT = '@'.charCodeAt(0)
  21. // SCSS PATCH {
  22. const COMMA = ','.charCodeAt(0)
  23. const HASH = '#'.charCodeAt(0)
  24. // } SCSS PATCH
  25. const RE_AT_END = /[\t\n\f\r "#'()/;[\\\]{}]/g
  26. const RE_WORD_END = /[,\t\n\f\r !"#'():;@[\\\]{}]|\/(?=\*)/g
  27. const RE_BAD_BRACKET = /.[\r\n"'(/\\]/
  28. const RE_HEX_ESCAPE = /[\da-f]/i
  29. const RE_NEW_LINE = /[\n\f\r]/g // SCSS PATCH
  30. // SCSS PATCH function name was changed
  31. module.exports = function scssTokenize(input, options = {}) {
  32. let css = input.css.valueOf()
  33. let ignore = options.ignoreErrors
  34. let code, next, quote, content, escape
  35. let escaped, prev, n, currentToken
  36. let length = css.length
  37. let pos = 0
  38. let buffer = []
  39. let returned = []
  40. let brackets // SCSS PATCH
  41. function position() {
  42. return pos
  43. }
  44. function unclosed(what) {
  45. throw input.error('Unclosed ' + what, pos)
  46. }
  47. function endOfFile() {
  48. return returned.length === 0 && pos >= length
  49. }
  50. // SCSS PATCH {
  51. function interpolation() {
  52. let deep = 1
  53. let stringQuote = false
  54. let stringEscaped = false
  55. while (deep > 0) {
  56. next += 1
  57. if (css.length <= next) unclosed('interpolation')
  58. code = css.charCodeAt(next)
  59. n = css.charCodeAt(next + 1)
  60. if (stringQuote) {
  61. if (!stringEscaped && code === stringQuote) {
  62. stringQuote = false
  63. stringEscaped = false
  64. } else if (code === BACKSLASH) {
  65. stringEscaped = !stringEscaped
  66. } else if (stringEscaped) {
  67. stringEscaped = false
  68. }
  69. } else if (code === SINGLE_QUOTE || code === DOUBLE_QUOTE) {
  70. stringQuote = code
  71. } else if (code === CLOSE_CURLY) {
  72. deep -= 1
  73. } else if (code === HASH && n === OPEN_CURLY) {
  74. deep += 1
  75. }
  76. }
  77. }
  78. // } SCSS PATCH
  79. function nextToken(opts) {
  80. if (returned.length) return returned.pop()
  81. if (pos >= length) return undefined
  82. let ignoreUnclosed = opts ? opts.ignoreUnclosed : false
  83. code = css.charCodeAt(pos)
  84. switch (code) {
  85. case NEWLINE:
  86. case SPACE:
  87. case TAB:
  88. case CR:
  89. case FEED: {
  90. next = pos
  91. do {
  92. next += 1
  93. code = css.charCodeAt(next)
  94. } while (
  95. code === SPACE ||
  96. code === NEWLINE ||
  97. code === TAB ||
  98. code === CR ||
  99. code === FEED
  100. )
  101. currentToken = ['space', css.slice(pos, next)]
  102. pos = next - 1
  103. break
  104. }
  105. case OPEN_SQUARE:
  106. case CLOSE_SQUARE:
  107. case OPEN_CURLY:
  108. case CLOSE_CURLY:
  109. case COLON:
  110. case SEMICOLON:
  111. case CLOSE_PARENTHESES: {
  112. let controlChar = String.fromCharCode(code)
  113. currentToken = [controlChar, controlChar, pos]
  114. break
  115. }
  116. // SCSS PATCH {
  117. case COMMA: {
  118. currentToken = ['word', ',', pos, pos + 1]
  119. break
  120. }
  121. // } SCSS PATCH
  122. case OPEN_PARENTHESES: {
  123. prev = buffer.length ? buffer.pop()[1] : ''
  124. n = css.charCodeAt(pos + 1)
  125. // SCSS PATCH {
  126. if (prev === 'url' && n !== SINGLE_QUOTE && n !== DOUBLE_QUOTE) {
  127. brackets = 1
  128. escaped = false
  129. next = pos + 1
  130. while (next <= css.length - 1) {
  131. n = css.charCodeAt(next)
  132. if (n === BACKSLASH) {
  133. escaped = !escaped
  134. } else if (n === OPEN_PARENTHESES) {
  135. brackets += 1
  136. } else if (n === CLOSE_PARENTHESES) {
  137. brackets -= 1
  138. if (brackets === 0) break
  139. }
  140. next += 1
  141. }
  142. content = css.slice(pos, next + 1)
  143. currentToken = ['brackets', content, pos, next]
  144. pos = next
  145. // } SCSS PATCH
  146. } else {
  147. next = css.indexOf(')', pos + 1)
  148. content = css.slice(pos, next + 1)
  149. if (next === -1 || RE_BAD_BRACKET.test(content)) {
  150. currentToken = ['(', '(', pos]
  151. } else {
  152. currentToken = ['brackets', content, pos, next]
  153. pos = next
  154. }
  155. }
  156. break
  157. }
  158. case SINGLE_QUOTE:
  159. case DOUBLE_QUOTE: {
  160. // SCSS PATCH {
  161. quote = code
  162. next = pos
  163. escaped = false
  164. while (next < length) {
  165. next++
  166. if (next === length) unclosed('string')
  167. code = css.charCodeAt(next)
  168. n = css.charCodeAt(next + 1)
  169. if (!escaped && code === quote) {
  170. break
  171. } else if (code === BACKSLASH) {
  172. escaped = !escaped
  173. } else if (escaped) {
  174. escaped = false
  175. } else if (code === HASH && n === OPEN_CURLY) {
  176. interpolation()
  177. }
  178. }
  179. // } SCSS PATCH
  180. currentToken = ['string', css.slice(pos, next + 1), pos, next]
  181. pos = next
  182. break
  183. }
  184. case AT: {
  185. RE_AT_END.lastIndex = pos + 1
  186. RE_AT_END.test(css)
  187. if (RE_AT_END.lastIndex === 0) {
  188. next = css.length - 1
  189. } else {
  190. next = RE_AT_END.lastIndex - 2
  191. }
  192. currentToken = ['at-word', css.slice(pos, next + 1), pos, next]
  193. pos = next
  194. break
  195. }
  196. case BACKSLASH: {
  197. next = pos
  198. escape = true
  199. while (css.charCodeAt(next + 1) === BACKSLASH) {
  200. next += 1
  201. escape = !escape
  202. }
  203. code = css.charCodeAt(next + 1)
  204. if (
  205. escape &&
  206. code !== SLASH &&
  207. code !== SPACE &&
  208. code !== NEWLINE &&
  209. code !== TAB &&
  210. code !== CR &&
  211. code !== FEED
  212. ) {
  213. next += 1
  214. if (RE_HEX_ESCAPE.test(css.charAt(next))) {
  215. while (RE_HEX_ESCAPE.test(css.charAt(next + 1))) {
  216. next += 1
  217. }
  218. if (css.charCodeAt(next + 1) === SPACE) {
  219. next += 1
  220. }
  221. }
  222. }
  223. currentToken = ['word', css.slice(pos, next + 1), pos, next]
  224. pos = next
  225. break
  226. }
  227. default:
  228. // SCSS PATCH {
  229. n = css.charCodeAt(pos + 1)
  230. if (code === HASH && n === OPEN_CURLY) {
  231. next = pos
  232. interpolation()
  233. content = css.slice(pos, next + 1)
  234. currentToken = ['word', content, pos, next]
  235. pos = next
  236. } else if (code === SLASH && n === ASTERISK) {
  237. // } SCSS PATCH
  238. next = css.indexOf('*/', pos + 2) + 1
  239. if (next === 0) {
  240. if (ignore || ignoreUnclosed) {
  241. next = css.length
  242. } else {
  243. unclosed('comment')
  244. }
  245. }
  246. currentToken = ['comment', css.slice(pos, next + 1), pos, next]
  247. pos = next
  248. // SCSS PATCH {
  249. } else if (code === SLASH && n === SLASH) {
  250. RE_NEW_LINE.lastIndex = pos + 1
  251. RE_NEW_LINE.test(css)
  252. if (RE_NEW_LINE.lastIndex === 0) {
  253. next = css.length - 1
  254. } else {
  255. next = RE_NEW_LINE.lastIndex - 2
  256. }
  257. content = css.slice(pos, next + 1)
  258. currentToken = ['comment', content, pos, next, 'inline']
  259. pos = next
  260. // } SCSS PATCH
  261. } else {
  262. RE_WORD_END.lastIndex = pos + 1
  263. RE_WORD_END.test(css)
  264. if (RE_WORD_END.lastIndex === 0) {
  265. next = css.length - 1
  266. } else {
  267. next = RE_WORD_END.lastIndex - 2
  268. }
  269. currentToken = ['word', css.slice(pos, next + 1), pos, next]
  270. buffer.push(currentToken)
  271. pos = next
  272. }
  273. break
  274. }
  275. pos++
  276. return currentToken
  277. }
  278. function back(token) {
  279. returned.push(token)
  280. }
  281. return {
  282. back,
  283. endOfFile,
  284. nextToken,
  285. position
  286. }
  287. }