1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
| #! /usr/bin/env python
|
| """Token constants (from "token.h")."""
|
| # Taken from Python (r53757) and modified to include some tokens
| # originally monkeypatched in by pgen2.tokenize
|
| #--start constants--
| ENDMARKER = 0
| NAME = 1
| NUMBER = 2
| STRING = 3
| NEWLINE = 4
| INDENT = 5
| DEDENT = 6
| LPAR = 7
| RPAR = 8
| LSQB = 9
| RSQB = 10
| COLON = 11
| COMMA = 12
| SEMI = 13
| PLUS = 14
| MINUS = 15
| STAR = 16
| SLASH = 17
| VBAR = 18
| AMPER = 19
| LESS = 20
| GREATER = 21
| EQUAL = 22
| DOT = 23
| PERCENT = 24
| BACKQUOTE = 25
| LBRACE = 26
| RBRACE = 27
| EQEQUAL = 28
| NOTEQUAL = 29
| LESSEQUAL = 30
| GREATEREQUAL = 31
| TILDE = 32
| CIRCUMFLEX = 33
| LEFTSHIFT = 34
| RIGHTSHIFT = 35
| DOUBLESTAR = 36
| PLUSEQUAL = 37
| MINEQUAL = 38
| STAREQUAL = 39
| SLASHEQUAL = 40
| PERCENTEQUAL = 41
| AMPEREQUAL = 42
| VBAREQUAL = 43
| CIRCUMFLEXEQUAL = 44
| LEFTSHIFTEQUAL = 45
| RIGHTSHIFTEQUAL = 46
| DOUBLESTAREQUAL = 47
| DOUBLESLASH = 48
| DOUBLESLASHEQUAL = 49
| AT = 50
| ATEQUAL = 51
| OP = 52
| COMMENT = 53
| NL = 54
| RARROW = 55
| ERRORTOKEN = 56
| N_TOKENS = 57
| NT_OFFSET = 256
| #--end constants--
|
| tok_name = {}
| for _name, _value in globals().items():
| if type(_value) is type(0):
| tok_name[_value] = _name
|
|
| def ISTERMINAL(x):
| return x < NT_OFFSET
|
| def ISNONTERMINAL(x):
| return x >= NT_OFFSET
|
| def ISEOF(x):
| return x == ENDMARKER
|
|
