9
3

llex.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565
  1. /*
  2. ** $Id: llex.c,v 2.96 2016/05/02 14:02:12 roberto Exp $
  3. ** Lexical Analyzer
  4. ** See Copyright Notice in lua.h
  5. */
  6. #define llex_c
  7. #define LUA_CORE
  8. #include "lprefix.h"
  9. #include <locale.h>
  10. #include <string.h>
  11. #include "lua.h"
  12. #include "lctype.h"
  13. #include "ldebug.h"
  14. #include "ldo.h"
  15. #include "lgc.h"
  16. #include "llex.h"
  17. #include "lobject.h"
  18. #include "lparser.h"
  19. #include "lstate.h"
  20. #include "lstring.h"
  21. #include "ltable.h"
  22. #include "lzio.h"
  23. #define next(ls) (ls->current = zgetc(ls->z))
  24. #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r')
  25. /* ORDER RESERVED */
  26. static const char *const luaX_tokens [] = {
  27. "and", "break", "do", "else", "elseif",
  28. "end", "false", "for", "function", "goto", "if",
  29. "in", "local", "nil", "not", "or", "repeat",
  30. "return", "then", "true", "until", "while",
  31. "//", "..", "...", "==", ">=", "<=", "~=",
  32. "<<", ">>", "::", "<eof>",
  33. "<number>", "<integer>", "<name>", "<string>"
  34. };
  35. #define save_and_next(ls) (save(ls, ls->current), next(ls))
  36. static l_noret lexerror (LexState *ls, const char *msg, int token);
  37. static void save (LexState *ls, int c) {
  38. Mbuffer *b = ls->buff;
  39. if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) {
  40. size_t newsize;
  41. if (luaZ_sizebuffer(b) >= MAX_SIZE/2)
  42. lexerror(ls, "lexical element too long", 0);
  43. newsize = luaZ_sizebuffer(b) * 2;
  44. luaZ_resizebuffer(ls->L, b, newsize);
  45. }
  46. b->buffer[luaZ_bufflen(b)++] = cast(char, c);
  47. }
  48. void luaX_init (lua_State *L) {
  49. int i;
  50. TString *e = luaS_newliteral(L, LUA_ENV); /* create env name */
  51. luaC_fix(L, obj2gco(e)); /* never collect this name */
  52. for (i=0; i<NUM_RESERVED; i++) {
  53. TString *ts = luaS_new(L, luaX_tokens[i]);
  54. luaC_fix(L, obj2gco(ts)); /* reserved words are never collected */
  55. ts->extra = cast_byte(i+1); /* reserved word */
  56. }
  57. }
  58. const char *luaX_token2str (LexState *ls, int token) {
  59. if (token < FIRST_RESERVED) { /* single-byte symbols? */
  60. lua_assert(token == cast_uchar(token));
  61. return luaO_pushfstring(ls->L, "'%c'", token);
  62. }
  63. else {
  64. const char *s = luaX_tokens[token - FIRST_RESERVED];
  65. if (token < TK_EOS) /* fixed format (symbols and reserved words)? */
  66. return luaO_pushfstring(ls->L, "'%s'", s);
  67. else /* names, strings, and numerals */
  68. return s;
  69. }
  70. }
  71. static const char *txtToken (LexState *ls, int token) {
  72. switch (token) {
  73. case TK_NAME: case TK_STRING:
  74. case TK_FLT: case TK_INT:
  75. save(ls, '\0');
  76. return luaO_pushfstring(ls->L, "'%s'", luaZ_buffer(ls->buff));
  77. default:
  78. return luaX_token2str(ls, token);
  79. }
  80. }
  81. static l_noret lexerror (LexState *ls, const char *msg, int token) {
  82. msg = luaG_addinfo(ls->L, msg, ls->source, ls->linenumber);
  83. if (token)
  84. luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token));
  85. luaD_throw(ls->L, LUA_ERRSYNTAX);
  86. }
  87. l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
  88. lexerror(ls, msg, ls->t.token);
  89. }
  90. /*
  91. ** creates a new string and anchors it in scanner's table so that
  92. ** it will not be collected until the end of the compilation
  93. ** (by that time it should be anchored somewhere)
  94. */
  95. TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
  96. lua_State *L = ls->L;
  97. TValue *o; /* entry for 'str' */
  98. TString *ts = luaS_newlstr(L, str, l); /* create new string */
  99. setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */
  100. o = luaH_set(L, ls->h, L->top - 1);
  101. if (ttisnil(o)) { /* not in use yet? */
  102. /* boolean value does not need GC barrier;
  103. table has no metatable, so it does not need to invalidate cache */
  104. setbvalue(o, 1); /* t[string] = true */
  105. luaC_checkGC(L);
  106. }
  107. else { /* string already present */
  108. ts = tsvalue(keyfromval(o)); /* re-use value previously stored */
  109. }
  110. L->top--; /* remove string from stack */
  111. return ts;
  112. }
  113. /*
  114. ** increment line number and skips newline sequence (any of
  115. ** \n, \r, \n\r, or \r\n)
  116. */
  117. static void inclinenumber (LexState *ls) {
  118. int old = ls->current;
  119. lua_assert(currIsNewline(ls));
  120. next(ls); /* skip '\n' or '\r' */
  121. if (currIsNewline(ls) && ls->current != old)
  122. next(ls); /* skip '\n\r' or '\r\n' */
  123. if (++ls->linenumber >= MAX_INT)
  124. lexerror(ls, "chunk has too many lines", 0);
  125. }
  126. void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source,
  127. int firstchar) {
  128. ls->t.token = 0;
  129. ls->L = L;
  130. ls->current = firstchar;
  131. ls->lookahead.token = TK_EOS; /* no look-ahead token */
  132. ls->z = z;
  133. ls->fs = NULL;
  134. ls->linenumber = 1;
  135. ls->lastline = 1;
  136. ls->source = source;
  137. ls->envn = luaS_newliteral(L, LUA_ENV); /* get env name */
  138. luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */
  139. }
  140. /*
  141. ** =======================================================
  142. ** LEXICAL ANALYZER
  143. ** =======================================================
  144. */
  145. static int check_next1 (LexState *ls, int c) {
  146. if (ls->current == c) {
  147. next(ls);
  148. return 1;
  149. }
  150. else return 0;
  151. }
  152. /*
  153. ** Check whether current char is in set 'set' (with two chars) and
  154. ** saves it
  155. */
  156. static int check_next2 (LexState *ls, const char *set) {
  157. lua_assert(set[2] == '\0');
  158. if (ls->current == set[0] || ls->current == set[1]) {
  159. save_and_next(ls);
  160. return 1;
  161. }
  162. else return 0;
  163. }
  164. /* LUA_NUMBER */
  165. /*
  166. ** this function is quite liberal in what it accepts, as 'luaO_str2num'
  167. ** will reject ill-formed numerals.
  168. */
  169. static int read_numeral (LexState *ls, SemInfo *seminfo) {
  170. TValue obj;
  171. const char *expo = "Ee";
  172. int first = ls->current;
  173. lua_assert(lisdigit(ls->current));
  174. save_and_next(ls);
  175. if (first == '0' && check_next2(ls, "xX")) /* hexadecimal? */
  176. expo = "Pp";
  177. for (;;) {
  178. if (check_next2(ls, expo)) /* exponent part? */
  179. check_next2(ls, "-+"); /* optional exponent sign */
  180. if (lisxdigit(ls->current))
  181. save_and_next(ls);
  182. else if (ls->current == '.')
  183. save_and_next(ls);
  184. else break;
  185. }
  186. save(ls, '\0');
  187. if (luaO_str2num(luaZ_buffer(ls->buff), &obj) == 0) /* format error? */
  188. lexerror(ls, "malformed number", TK_FLT);
  189. if (ttisinteger(&obj)) {
  190. seminfo->i = ivalue(&obj);
  191. return TK_INT;
  192. }
  193. else {
  194. lua_assert(ttisfloat(&obj));
  195. seminfo->r = fltvalue(&obj);
  196. return TK_FLT;
  197. }
  198. }
  199. /*
  200. ** skip a sequence '[=*[' or ']=*]'; if sequence is well formed, return
  201. ** its number of '='s; otherwise, return a negative number (-1 iff there
  202. ** are no '='s after initial bracket)
  203. */
  204. static int skip_sep (LexState *ls) {
  205. int count = 0;
  206. int s = ls->current;
  207. lua_assert(s == '[' || s == ']');
  208. save_and_next(ls);
  209. while (ls->current == '=') {
  210. save_and_next(ls);
  211. count++;
  212. }
  213. return (ls->current == s) ? count : (-count) - 1;
  214. }
  215. static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
  216. int line = ls->linenumber; /* initial line (for error message) */
  217. save_and_next(ls); /* skip 2nd '[' */
  218. if (currIsNewline(ls)) /* string starts with a newline? */
  219. inclinenumber(ls); /* skip it */
  220. for (;;) {
  221. switch (ls->current) {
  222. case EOZ: { /* error */
  223. const char *what = (seminfo ? "string" : "comment");
  224. const char *msg = luaO_pushfstring(ls->L,
  225. "unfinished long %s (starting at line %d)", what, line);
  226. lexerror(ls, msg, TK_EOS);
  227. break; /* to avoid warnings */
  228. }
  229. case ']': {
  230. if (skip_sep(ls) == sep) {
  231. save_and_next(ls); /* skip 2nd ']' */
  232. goto endloop;
  233. }
  234. break;
  235. }
  236. case '\n': case '\r': {
  237. save(ls, '\n');
  238. inclinenumber(ls);
  239. if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */
  240. break;
  241. }
  242. default: {
  243. if (seminfo) save_and_next(ls);
  244. else next(ls);
  245. }
  246. }
  247. } endloop:
  248. if (seminfo)
  249. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
  250. luaZ_bufflen(ls->buff) - 2*(2 + sep));
  251. }
  252. static void esccheck (LexState *ls, int c, const char *msg) {
  253. if (!c) {
  254. if (ls->current != EOZ)
  255. save_and_next(ls); /* add current to buffer for error message */
  256. lexerror(ls, msg, TK_STRING);
  257. }
  258. }
  259. static int gethexa (LexState *ls) {
  260. save_and_next(ls);
  261. esccheck (ls, lisxdigit(ls->current), "hexadecimal digit expected");
  262. return luaO_hexavalue(ls->current);
  263. }
  264. static int readhexaesc (LexState *ls) {
  265. int r = gethexa(ls);
  266. r = (r << 4) + gethexa(ls);
  267. luaZ_buffremove(ls->buff, 2); /* remove saved chars from buffer */
  268. return r;
  269. }
  270. static unsigned long readutf8esc (LexState *ls) {
  271. unsigned long r;
  272. int i = 4; /* chars to be removed: '\', 'u', '{', and first digit */
  273. save_and_next(ls); /* skip 'u' */
  274. esccheck(ls, ls->current == '{', "missing '{'");
  275. r = gethexa(ls); /* must have at least one digit */
  276. while ((save_and_next(ls), lisxdigit(ls->current))) {
  277. i++;
  278. r = (r << 4) + luaO_hexavalue(ls->current);
  279. esccheck(ls, r <= 0x10FFFF, "UTF-8 value too large");
  280. }
  281. esccheck(ls, ls->current == '}', "missing '}'");
  282. next(ls); /* skip '}' */
  283. luaZ_buffremove(ls->buff, i); /* remove saved chars from buffer */
  284. return r;
  285. }
  286. static void utf8esc (LexState *ls) {
  287. char buff[UTF8BUFFSZ];
  288. int n = luaO_utf8esc(buff, readutf8esc(ls));
  289. for (; n > 0; n--) /* add 'buff' to string */
  290. save(ls, buff[UTF8BUFFSZ - n]);
  291. }
  292. static int readdecesc (LexState *ls) {
  293. int i;
  294. int r = 0; /* result accumulator */
  295. for (i = 0; i < 3 && lisdigit(ls->current); i++) { /* read up to 3 digits */
  296. r = 10*r + ls->current - '0';
  297. save_and_next(ls);
  298. }
  299. esccheck(ls, r <= UCHAR_MAX, "decimal escape too large");
  300. luaZ_buffremove(ls->buff, i); /* remove read digits from buffer */
  301. return r;
  302. }
  303. static void read_string (LexState *ls, int del, SemInfo *seminfo) {
  304. save_and_next(ls); /* keep delimiter (for error messages) */
  305. while (ls->current != del) {
  306. switch (ls->current) {
  307. case EOZ:
  308. lexerror(ls, "unfinished string", TK_EOS);
  309. break; /* to avoid warnings */
  310. case '\n':
  311. case '\r':
  312. lexerror(ls, "unfinished string", TK_STRING);
  313. break; /* to avoid warnings */
  314. case '\\': { /* escape sequences */
  315. int c; /* final character to be saved */
  316. save_and_next(ls); /* keep '\\' for error messages */
  317. switch (ls->current) {
  318. case 'a': c = '\a'; goto read_save;
  319. case 'b': c = '\b'; goto read_save;
  320. case 'f': c = '\f'; goto read_save;
  321. case 'n': c = '\n'; goto read_save;
  322. case 'r': c = '\r'; goto read_save;
  323. case 't': c = '\t'; goto read_save;
  324. case 'v': c = '\v'; goto read_save;
  325. case 'x': c = readhexaesc(ls); goto read_save;
  326. case 'u': utf8esc(ls); goto no_save;
  327. case '\n': case '\r':
  328. inclinenumber(ls); c = '\n'; goto only_save;
  329. case '\\': case '\"': case '\'':
  330. c = ls->current; goto read_save;
  331. case EOZ: goto no_save; /* will raise an error next loop */
  332. case 'z': { /* zap following span of spaces */
  333. luaZ_buffremove(ls->buff, 1); /* remove '\\' */
  334. next(ls); /* skip the 'z' */
  335. while (lisspace(ls->current)) {
  336. if (currIsNewline(ls)) inclinenumber(ls);
  337. else next(ls);
  338. }
  339. goto no_save;
  340. }
  341. default: {
  342. esccheck(ls, lisdigit(ls->current), "invalid escape sequence");
  343. c = readdecesc(ls); /* digital escape '\ddd' */
  344. goto only_save;
  345. }
  346. }
  347. read_save:
  348. next(ls);
  349. /* go through */
  350. only_save:
  351. luaZ_buffremove(ls->buff, 1); /* remove '\\' */
  352. save(ls, c);
  353. /* go through */
  354. no_save: break;
  355. }
  356. default:
  357. save_and_next(ls);
  358. }
  359. }
  360. save_and_next(ls); /* skip delimiter */
  361. seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
  362. luaZ_bufflen(ls->buff) - 2);
  363. }
  364. static int llex (LexState *ls, SemInfo *seminfo) {
  365. luaZ_resetbuffer(ls->buff);
  366. for (;;) {
  367. switch (ls->current) {
  368. case '\n': case '\r': { /* line breaks */
  369. inclinenumber(ls);
  370. break;
  371. }
  372. case ' ': case '\f': case '\t': case '\v': { /* spaces */
  373. next(ls);
  374. break;
  375. }
  376. case '-': { /* '-' or '--' (comment) */
  377. next(ls);
  378. if (ls->current != '-') return '-';
  379. /* else is a comment */
  380. next(ls);
  381. if (ls->current == '[') { /* long comment? */
  382. int sep = skip_sep(ls);
  383. luaZ_resetbuffer(ls->buff); /* 'skip_sep' may dirty the buffer */
  384. if (sep >= 0) {
  385. read_long_string(ls, NULL, sep); /* skip long comment */
  386. luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */
  387. break;
  388. }
  389. }
  390. /* else short comment */
  391. while (!currIsNewline(ls) && ls->current != EOZ)
  392. next(ls); /* skip until end of line (or end of file) */
  393. break;
  394. }
  395. case '[': { /* long string or simply '[' */
  396. int sep = skip_sep(ls);
  397. if (sep >= 0) {
  398. read_long_string(ls, seminfo, sep);
  399. return TK_STRING;
  400. }
  401. else if (sep != -1) /* '[=...' missing second bracket */
  402. lexerror(ls, "invalid long string delimiter", TK_STRING);
  403. return '[';
  404. }
  405. case '=': {
  406. next(ls);
  407. if (check_next1(ls, '=')) return TK_EQ;
  408. else return '=';
  409. }
  410. case '<': {
  411. next(ls);
  412. if (check_next1(ls, '=')) return TK_LE;
  413. else if (check_next1(ls, '<')) return TK_SHL;
  414. else return '<';
  415. }
  416. case '>': {
  417. next(ls);
  418. if (check_next1(ls, '=')) return TK_GE;
  419. else if (check_next1(ls, '>')) return TK_SHR;
  420. else return '>';
  421. }
  422. case '/': {
  423. next(ls);
  424. if (check_next1(ls, '/')) return TK_IDIV;
  425. else return '/';
  426. }
  427. case '~': {
  428. next(ls);
  429. if (check_next1(ls, '=')) return TK_NE;
  430. else return '~';
  431. }
  432. case ':': {
  433. next(ls);
  434. if (check_next1(ls, ':')) return TK_DBCOLON;
  435. else return ':';
  436. }
  437. case '"': case '\'': { /* short literal strings */
  438. read_string(ls, ls->current, seminfo);
  439. return TK_STRING;
  440. }
  441. case '.': { /* '.', '..', '...', or number */
  442. save_and_next(ls);
  443. if (check_next1(ls, '.')) {
  444. if (check_next1(ls, '.'))
  445. return TK_DOTS; /* '...' */
  446. else return TK_CONCAT; /* '..' */
  447. }
  448. else if (!lisdigit(ls->current)) return '.';
  449. else return read_numeral(ls, seminfo);
  450. }
  451. case '0': case '1': case '2': case '3': case '4':
  452. case '5': case '6': case '7': case '8': case '9': {
  453. return read_numeral(ls, seminfo);
  454. }
  455. case EOZ: {
  456. return TK_EOS;
  457. }
  458. default: {
  459. if (lislalpha(ls->current)) { /* identifier or reserved word? */
  460. TString *ts;
  461. do {
  462. save_and_next(ls);
  463. } while (lislalnum(ls->current));
  464. ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
  465. luaZ_bufflen(ls->buff));
  466. seminfo->ts = ts;
  467. if (isreserved(ts)) /* reserved word? */
  468. return ts->extra - 1 + FIRST_RESERVED;
  469. else {
  470. return TK_NAME;
  471. }
  472. }
  473. else { /* single-char tokens (+ - / ...) */
  474. int c = ls->current;
  475. next(ls);
  476. return c;
  477. }
  478. }
  479. }
  480. }
  481. }
  482. void luaX_next (LexState *ls) {
  483. ls->lastline = ls->linenumber;
  484. if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */
  485. ls->t = ls->lookahead; /* use this one */
  486. ls->lookahead.token = TK_EOS; /* and discharge it */
  487. }
  488. else
  489. ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */
  490. }
  491. int luaX_lookahead (LexState *ls) {
  492. lua_assert(ls->lookahead.token == TK_EOS);
  493. ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
  494. return ls->lookahead.token;
  495. }