a

�DOgR�
@s�dZd
Z
ddlZddlZddlmZmZ
ddlTddlm	Z	
dd	�e
e	�
D�
gd
�
Z[	ze
Wne
yv


eZYn0dd�Zd
d�Zdd�Zdd�ZdZdZeede�
ee�
ZdZdZdZdZedd�Zeeeee�ZdZedd�ee�
ZdeZeee�Zed ed!�Z ee ee�Z!d"Z"d#Z#d$Z$d%Z%d&Z&ee&d'e&d(�Z'ee&d)e&d*�Z(ed+d,d-d.d/d0d1d2d3�	Z)d4Z*ed5d6d7�Z+ee)e*e+�Z,ee!e,e(e�Z-ee-Z.ee&d8ed9d�e&d:ed;d��Z/edee'�Z0eee0e!e,e/e�Z1e2ej3e.e1e$e%f�\Z4Z5Z6Z7ed<d=d>d?�ed<d=d@dA�BhdB�
BZ8e�3e"�
e�3e#�
e6e7dC�dDdE�e8D�
�
dFdE�e8D�
�
dGdE�e8D�
�
Z9d'd(hdHdI�e8D�
BdJdI�e8D�
BZ:d9d;hdKdI�e8D�
BdLdI�e8D�
BZ;dMZ<GdNdO�dOe=�Z>GdPdQ�dQe=�Z?dRdS�Z@e@f
dTdU�
ZAdVdW�ZBGdXdY�dY�ZCe�3dZejD�ZEe�3d[ejD�ZFd\d]�ZGd^d_�ZHd`da�ZIdbdc�ZJeKddk�r�ddlLZLeMeLjN�
dk�rxeAeOeLjNd�
jP�

neAeLjQjP�

dS)ezKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro�N)�BOM_UTF8�lookup)
�
*�
)
�tokenc
Csg|]}
|
dd
kr|
�qS)r
�
_�)�.0�
xrr�./usr/lib64/python3.9/lib2to3/pgen2/tokenize.py�
<listcomp>%�r)�tokenize�generate_tokens�
untokenizec
Gsd
d�|�
dS)N�
(�
|�
))
�join�
�choicesrrr�group0r
rc
Gst|�d
S)Nr�
rrrrr�any1r
rc
Gst|�d
S)N�
?rrrrr�maybe2r
rc
st�f
d
d��D�
�
S)Nc
3s4|],}
�dD]}|
��|��kr|
|V
qqd
S))
�N)
�casefold)r	r
�
y�
�
lrr�	<genexpr>4s
z _combinations.<locals>.<genexpr>)
�setrrrr�
_combinations3s

�r#z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z\d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?�'''�"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�
~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�
'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�
"�
r�
R�
f�
F�
b�
B>�
uZUR�
U�urZuRZUr)r'r(r$r%c
Csi|]}
|
�d�t�qS�
r$)
�single3prog�r	�prefixrrr�
<dictcomp>yr
r6c
Csi|]}
|
�d�t�qS�
r%)
�double3progr4rrrr6zr
c
Csi|]
}
|
d�qS�
Nrr4rrrr6{r
c
Csh|]}
|
�d��qSr2rr4rrr�	<setcomp>r
r:c
Csh|]}
|
�d��qSr7rr4rrrr:�r
c
Csh|]}
|
�d��qS)
r'rr4rrrr:�r
c
Csh|]}
|
�d��qS)
r(rr4rrrr:�r
�c
@seZ
dZd
S)�
TokenErrorN��__name__�
__module__�__qualname__rrrrr<�r
r<c
@seZ
dZd
S)�StopTokenizingNr=rrrrrA�r
rAc		Cs4|\}}|\}}td
||||t
|t|
�
f�

dS)Nz%d,%d-%d,%d:	%s	%s)�print�tok_name�repr)	�typerZxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolrrr�
printtoken�s




�rGcCs&zt||
�
Wnt
y 


Yn0dSr9)�
tokenize_looprA)�readline�
tokeneaterrrrr�s



rcCst|�
D]}|
|�
qdSr9)
r)rIrJZ
token_inforrrrH�s

rHc@s,eZ
dZd
d�Zdd�Zdd�Zdd�Zd	S)
�Untokenizerc

Csg|_d
|_
d|_dS)Nrr
)�tokens�prev_row�prev_col)
�selfrrr�__init__�s


zUntokenizer.__init__cCs*|
\}}||j}|r&|j
�d
|�

dS)N�
 )rNrL�append)rO�start�row�col�
col_offsetrrr�add_whitespace�s



zUntokenizer.add_whitespacecCs�|
D]p}t|�
d
kr$|�
||
�

qv|\}}}}}|�|�

|j�|�

|\|_|_|ttfvr|jd7_d|_qd�	|j�
S)N�rr
r)
�len�compatrWrLrRrMrN�NEWLINE�NLr)rO�iterable�
t�tok_typerrS�endrFrrrr�s












zUntokenizer.untokenizec	Cs�d
}g}|jj
}|
\}}|ttfvr,|d7}|ttfvr<d}|D]�}|dd�\}}|ttttfvrl|d7}|tkr�|�
|�

q@n>|t	kr�|�
�
q@n*|ttfvr�d}n|r�|r�||d�

d
}||�

q@dS)NFrQTrX���)rLrR�NAME�NUMBERr[r\�ASYNC�AWAIT�INDENT�DEDENT�pop)	rOrr]�	startline�indents�toks_append�toknum�tokval�tokrrrrZ�s0






















zUntokenizer.compatN)r>r?r@rPrWrrZrrrrrK�srKz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)c
CsH|dd
����
dd�}
|
dks*|
�d�
r.dS|
dvs@|
�d�
rDdS|S)	N�r�
-�utf-8zutf-8-)zlatin-1�
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-rr)�lower�replace�
startswith)�orig_enc�encrrr�_get_normal_name�s



�
rxc
s�d
�d}
d}�
f
dd�}�f
dd�}|�}|�t
�
rHd�|dd�}d	}|sT|gfS||�
}
|
rj|
|g
fSt�|�
s~||g
fS|�}|s�||g
fS||�
}
|
r�|
||gfS|||gfS)
NFrqcs&z��WSty 


t
�YS0dSr9)�
StopIteration�bytesr)
rIrr�read_or_stop
s



z%detect_encoding.<locals>.read_or_stopc
s�z|�d
�
}
Wnt
y"


YdS0t�|
�
}|s6dSt|�d�
�
}zt|�
}Wntyn


td|�
�
Yn0�r�|j	dkr�td�
�
|d7}|S)N�asciirzunknown encoding: rqzencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError�	cookie_re�matchrxrr�LookupError�SyntaxError�name)rF�line_stringr��encoding�codec)
�	bom_foundrr�find_cookie
s"















z$detect_encoding.<locals>.find_cookieT�z	utf-8-sig)rur�blank_rer�)rIr��defaultr{r��first�secondr)r�rIr�detect_encoding�s0





















r�c
Cst�}
|
�
|�
Sr9)rKr)r]�utrrrr:
s
rc
cs|d
}
}}d\}}d}d
g
}d}d}	d
}
d}z
|�}WntyN


d}Yn0|
d}
d
t
|�
}
}|�
r0|sztd|��
|�|�
}|r�|�d
�
}
}t||d|�||
|f||fV
d\}}d}nd|�
r|dd�dk�
r|d	d�d
k�
rt||||
t
|�
f|fV
d}d}q.n||}||}q.�nB|d
k�rZ|�sZ|�
sJ�q$d
}|
|k�
r�||
dk�
rp|d}n8||
dk�
r�|tdt}n||
d
k�
r�d
}n�
q�|
d}
�
qN|
|k�
rq$|�
r�|V
d}||
dv�r�||
dk�rR||
d��d�
}|
t
|�
}t	||
|
f|
|
t
|�
f|fV
t
||d�|
|f|
t
|�
f|fV
q.t
t	f||
dk||
d�|
|
f|
t
|�
f|fV
q.||dk�r�|�|�

t|d|
�|
d
f|
|
f|fV
||dk�r2||v
�r�t
dd|
|
|f��
|dd�}|	�r|
|dk�rd}	d}d
}
td|
|
f|
|
f|fV
�q�|	�rr|�rr|
|dk�rrd}	d}d
}
n|�sntd|
d
f��
d
}|
|kr.t�||
�}|�r�|�d�
\}}|
|f|
|f|}}}
|||�||}}|tjv�s�|dk�r�|dk�r�t||||fV
�q|dv�rHt}|d
k�rt
}n
|	�r$d}|�r4|V
d}|||||fV
�q|dk�rv|�rb|V
d}t	||||fV
�q|tv�r�t|}|�||
�}|�r�|�d
�
}
|||
�}|�r�|V
d}t|||
|
f|fV
n|
|f}||d�}|}q.�q|tv�s"|dd�tv�s"|dd�tv�r�|ddk�rv|
|f}t|�pZt|d�pZt|d}||d�d}}|}q.n |�r�|V
d}t||||fV
�q|���rz|dv�r�|	�r�|dk�r�tnt||||fV
�qrt||||f}|dk�r�|�s�|}�qr|dv�rb|�rb|d
tk�rb|ddk�rb|dk�r>d}	|d}
t|d|d|d|dfV
d}|�rr|V
d}|V
nz|dk�r�|�r�|V
d}t
|||
|
f|fV
d}nF|d v�r�|d}n|d!v�r�|d}|�r�|V
d}t||||fV
n(t||
|
|
f|
|
df|fV
|
d}
�qrq.|�r4|V
d}|dd�D]}td|
d
f|
d
fdfV
�q@td|
d
f|
d
fdfV
dS)"Nr
)rr
FrrzEOF in multi-line string���z\
���z\
rQ�
	�
z#
�
#z
raz3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�
.TrXr��

)�async�awaitr�)�def�forr���
\z([{z)]})ryrYr<r�r`�STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr\rRrf�IndentationErrorrg�
pseudoprog�span�string�digitsrcr[�
triple_quoted�endprogs�
single_quoted�isidentifierrdrerb�OP�	ENDMARKER)rI�lnum�parenlev�	continued�contstr�needcont�contlinerjZstashedZ	async_defZasync_def_indentZasync_def_nlrF�pos�max�strstart�endprog�endmatchr`�column�
comment_tokenZnl_pos�pseudomatchrS�spos�eposr�initial�newlinern�indentrrrrO
s�






















�

*

�










 










�
�
�


 






�













�
�

































��



�











�






�
�




�













�



r�__main__)R�
__author__�__credits__r��re�codecsrrZlib2to3.pgen2.tokenrr�dir�__all__rz�	NameError�strrrrr#�
Whitespace�Comment�Ignore�Name�	Binnumber�	Hexnumber�	Octnumber�	Decnumber�	Intnumber�Exponent�
Pointfloat�Expfloat�Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�StringZOperatorZBracket�Special�Funny�
PlainToken�Token�ContStr�PseudoExtras�PseudoToken�map�compileZ	tokenprogr�r3r8Z_strprefixesr�r�r�r��	Exceptionr<rArGrrHrK�ASCIIrr�rxr�rrr>�sysrY�argv�openrI�stdinrrrr�<module>s��


























�


�



�
��

�
���
����
���
���8
Ic


&