!A?hsk @ d Z dZddlZddlZddlZddlZddlZddlZddlZddl m
Z
ddl m Z m
Z
mZ ddlmZ ddlmZmZ dd lmZ dd Z G d d
e Zd Zd Zd Zd Z d Z G d de Z G d de Z G d de Z! G d de Z"dS ) zRefactoring framework.
Used as a main program, this can refactor any number of files and/or
recursively descend down directories. Imported as a module, this
provides infrastructure to write your own refactoring tool.
z#Guido van Rossum <guido@python.org> N)chain )drivertokenizetoken) find_root)pytreepygram) btm_matcherTc t | g g dg }g }t j |j D ]<\ }}}| d r!|r
|dd }| | =|S )zEReturn a sorted list of all available fix names in the given package.*fix_ N)
__import__pkgutil iter_modules__path__
startswithappend) fixer_pkg
remove_prefixpkg fix_namesfindernameispkgs =/opt/alt/python-internal/lib64/python3.11/lib2to3/refactor.pyget_all_fix_namesr s
YB
.
.CI & 3CL A A # #e ??6 " " #
ABBx T " " " c e Zd ZdS )
_EveryNodeN__name__
__module__ __qualname__ r r r! r! + Dr r! c t | t j t j f r| j t
| j hS t | t j r"| j rt | j S t
t | t j rAt }| j D ])}|D ]$}| t | %*|S t d| z )zf Accepts a pytree Pattern Node and returns a set
of the pattern types which will match first. Nz$Oh no! I don't understand pattern %s)
isinstancer NodePattern LeafPatterntyper! NegatedPatterncontent_get_head_typesWildcardPatternsetupdate Exception)patrpxs r r/ r/ / s #*F,>?@@ 8 z#v,-- ; 0"3;// /#v-.. EE - -A
-
- ++,,,,
-
:SA
B
BBr c Z t j t }g }| D ]}|j r[ t |j }|D ] }|| | ?# t $ r | | Y `w xY w|j !||j | | | t t j
j t j
j
D ] }|| | t | S )z^ Accepts a list of fixers and returns a dictionary
of head node type --> fixer list. ) collections defaultdictlistpatternr/ r r! _accept_typer r
python_grammar
symbol2numbervaluestokensextenddict)
fixer_list
head_nodeseveryfixerheads node_types r _get_headnode_dictrJ K sL (..J E
$
$ = $
8'
66 "' 8 8I y)0077778
$
$
$ U#####
$ !-5-.55e<<<< U####60>EEGG!079 9 , , 9 $$U++++
s A A?>A?c < fdt d D S )zN
Return the fully qualified names for fixers in the package pkg_name.
c g | ]
}d z |z S .r& ).0fix_namepkg_names r
<listcomp>z+get_fixers_from_package.<locals>.<listcomp>h s8 @ @ @
sNX % @ @ @r F)r )rQ s `r get_fixers_from_packagerS d s@ @ @ @ @ -h > > @ @ @ @r c | S Nr& )objs r _identityrW k s Jr c h d}t j t j | j fd}t t j t j t j h }t } | \ }}||v r|t j k r|rnd}n|t j k r|dk r | \ }}|t j k s|dk rn | \ }}|t j k s|dk rn | \ }}|t j
k r|dk r
| \ }}|t j k rV| | | \ }}|t j
k s|dk rn | \ }}|t j k Vnn n# t $ r Y nw xY wt | S ) NFc B t } | d | d fS )Nr r )next)tokgens r advancez(_detect_future_features.<locals>.advancer s 3ii1vs1v~ r Tfrom
__future__import(,)r generate_tokensioStringIOreadline frozensetr NEWLINENLCOMMENTr1 STRINGNAMEOPadd
StopIteration)sourcehave_docstringr] ignorefeaturestpvaluer\ s @r _detect_future_featuresrv o s N
"2;v#6#6#?
@
@C
x{EMB
C
CFuuH
IBV||u|##! !%uz!!evoo #GII E##u '<'< #GII E##u'8'8 #GII E>>esll ' IBEJ&& LL''' ' IBUX~~# ' IB EJ&& 3 4
X s 3D!F
F"!F"c e Zd ZdZdS )
FixerErrorz A fixer could not be loaded.N)r# r$ r% __doc__r& r r rx rx s &&&&r rx c e Zd ZddddZdZdZd dZd Zd Zd Z d
Z
d Z d d Z d d
Z
d Zd dZd Zd dZd Zd Z d!dZd"dZdZdZd Zd Zd Zd Zd Z d Z dS )#RefactoringToolF)print_function
exec_functionwrite_unchanged_filesFixr Nc P || _ |pg | _ | j | _ || j | t j | _ | j d r| j j d= n| j d r
| j j d= | j
d | _ g | _ t j d | _ g | _ d| _ t% j | j t( j | j | _ | \ | _ | _ g | _ t5 j | _ g | _ g | _ t? | j | j D ]k}|j r| j ! | $|| j v r| j " | H|| j v r| j " | ltG | j | _$ tG | j | _% dS )
zInitializer.
Args:
fixer_names: a list of fixers to import
options: a dict with configuration.
explicit: a list of fixers to run even if they are explicit.
Nr| printr} execr~ r{ F)convertlogger)&fixersexplicit_default_optionscopyoptionsr2 r
r> grammarkeywordsgetr~ errorslogging getLoggerr fixer_logwroter Driverr r
get_fixers pre_order
post_orderfilesbm
BottomMatcherBM
bmi_pre_orderbmi_post_orderr
BM_compatible add_fixerr rJ bmi_pre_order_headsbmi_post_order_heads)self fixer_namesr r rG s r __init__zRefactoringTool.__init__ s " B
,1133 L ( ( ( ,1133 <( ) . %g..
\/
* . %f-
&*\%5%56M%N%N" '(9::
mDL,2N+/;8 8 8 +///*;*;'
"$$ 4?DN;; 2 2E"
2!!%(((( $.(("))%0000$/))#**5111#5d6H#I#I $6t7J$K$K!!!r c g }g }| j D ]}t |i i dg }| dd d }| | j r |t | j d }| d }| j d d |D z } t || }n$# t $ r t d |d| dw xY w || j | j
} | j r*| j d
ur!|| j vr| d | !| d | | j d
k r| | Y| j dk r| | {t d| j z t' j d }
| |
| |
||fS )a Inspects the options to load the requested patterns and handlers.
Returns:
(pre_order, post_order), where pre_order is the list of fixers that
want a pre-order AST traversal, and post_order is the list that want
post-order traversal.
r
rN r N_ c 6 g | ]}| S r& )title)rO r6 s r rR z.RefactoringTool.get_fixers.<locals>.<listcomp> s 5O5O5OAaggii5O5O5Or z Can't find TzSkipping optional fixer: %szAdding transformation: %sprepostzIllegal fixer order: %r run_orderkey)r r rsplitr FILE_PREFIXlensplit CLASS_PREFIXjoingetattrAttributeErrorrx r r r log_message log_debugorderr operator
attrgettersort) r pre_order_fixerspost_order_fixers fix_mod_pathmodrP parts
class_name fix_classrG key_funcs r r zRefactoringTool.get_fixers s" K J JL \2rC599C#**32226H""4#344
<#C(8$9$9$:$:; NN3''E *RWW5O5O5O5O5O-P-PPJ
X #C 4 4 !
X
X
X jxxx!LMMSWW
X IdlDN;;E~
$-t";";
55 !>III NN6 A A A{e## ''....&&!((//// !:U[!HIII&{33 ( +++ 8 ,,, "344s 1C!C#c )z Called when an error occurs.r& )r msgargskwdss r log_errorzRefactoringTool.log_error s
r c H |r||z }| j | dS )zHook to log a message.N)r infor r r s r r zRefactoringTool.log_message s/ *C r c H |r||z }| j | d S rU )r debugr s r r zRefactoringTool.log_debug
s/ *C # r c dS )zTCalled with the old version, new version, and filename of a
refactored file.Nr& )r old_textnew_textfilenameequals r print_outputz RefactoringTool.print_output s
r c |D ]P}t j | r| ||| 9| ||| QdS )z)Refactor a list of files and directories.N)ospathisdir refactor_dir
refactor_file)r itemswrite
doctests_only dir_or_files r refactorzRefactoringTool.refactor sn ! F FKw}}[))
F!!+umDDDD"";}EEEE F Fr c t j dz }t j | D ]\ }}}| d| | | |D ]w}| d s`t j | d |k r7t j || } | | || xd |D |dd< dS )zDescends down a directory and refactor every Python file found.
Python files are assumed to have a .py extension.
Files and subdirectories starting with '.' are skipped.
pyzDescending into %srN r c < g | ]}| d |S rM )r )rO dns r rR z0RefactoringTool.refactor_dir.<locals>.<listcomp>2 s) KKK"
c8J8JK2KKKr N)
r extsepwalkr r r r splitextr r )
r dir_namer r py_extdirpathdirnames filenamesr fullnames
r r z RefactoringTool.refactor_dir s T!,.GH,=,=
L
L (GXy NN/ 9 9 9 MMOOO NN !
G
G ,, GG$$T**1-77 !w||GT : :H&&x
FFFKKKKKHQQQKK
L
Lr c t |d }n/# t $ r"}| d|| Y d}~dS d}~ww xY w t j |j d }| n# | w xY wt j |d|d 5 }| |fcddd S # 1 swxY w Y dS ) zG
Do our best to decode a Python source file correctly.
rbzCan't open %s: %sNNNr r5 r encodingnewline) openOSErrorr r detect_encodingrf closerd read)r r ferrr s r _read_python_sourcez#RefactoringTool._read_python_source4 s" Xt$$AA NN.# > > > ::::: /
;;A>H
GGIIIIAGGIIII
WXsXr
B
B
B &a6688X% & & & & & & & & & & & & & & & & & &s.
?:? A7 7B
(C CCc | | \ }}|dS |dz
}|rl| d| | || }| j s||k r| ||||| dS | d| dS | || }| j s |r7|j r0| t | dd ||| dS | d| dS )zRefactors a file.N
zRefactoring doctests in %szNo doctest changes in %sr )r r zNo changes in %s)r r refactor_docstringr~ processed_filerefactor_string was_changedstr)r r r r inputr outputtrees r r z RefactoringTool.refactor_fileD s@ 228<<x = F
= NN7 B B B,,UH==F)
EVu__##FHeUHMMMMM 98DDDDD''x88D)
=d
=t7G
=##CIIcrcNH*/( $ D D D D D 18<<<<<r c t | }d|v rt j | j _ | j | }nK# t $ r>}| d||j j | Y d}~| j | j _ dS d}~ww xY w | j | j _ n# | j | j _ w xY w||_
| d| | || |S )aF Refactor a given input string.
Args:
data: a string holding the code to be refactored.
name: a human-readable name for use in error/log messages.
Returns:
An AST corresponding to the refactored input stream; None if
there were errors during the parse.
r| zCan't parse %s: %s: %sNzRefactoring %s)
rv r
!python_grammar_no_print_statementr r parse_stringr3 r __class__r# future_featuresr
refactor_tree)r datar rs r r s r r z RefactoringTool.refactor_string[ s
+400 x ' '"("JDK /;++D11DD NN3 !7
>
>
> FFF"&,DK
#',DK $,DK . . . . ' '... 4&&& s/ A B$
B"B 2B$ BB$ $B7c t j }|rh| d | |d }| j s||k r| |d| d S | d d S | |d }| j s |r-|j r&| t | d| d S | d d S )Nz Refactoring doctests in stdinz<stdin>zNo doctest changes in stdinzNo changes in stdin)
sysstdinr r r r~ r r r r )r r r r r s r refactor_stdinz RefactoringTool.refactor_stdinv s 6 NN: ; ; ;,,UI>>F)
>Vu__##FIu===== <=====''y99D)
6d
6t7G
6##CIIy%@@@@@ 455555r c t | j | j D ]}| || | | j | | | j | | j | }t | r| j j D ]}||v r|| r||
t j j d |j r+||
t j j t' || D ]8}||| v r|| | t+ | n# t, $ r Y Ew xY w|j r
||j v rZ| | }|r| || }|| | | D ]*}|j sg |_ |j | +| j | }|D ],} | |vrg || < || || -:t | t | j | j D ]}| || |j S )a Refactors a parse tree (modifying the tree in place).
For compatible patterns the bottom matcher module is
used. Otherwise the tree is traversed node-to-node for
matches.
Args:
tree: a pytree.Node instance representing the root of the tree
to be refactored.
name: a human-readable name for this tree.
Returns:
True if the tree was modified, False otherwise.
T)r reverser ) r r r
start_tree traverse_byr r r runleavesanyr@ r r r Basedepthkeep_line_order
get_linenor; remover
ValueErrorfixers_appliedmatch transformreplacer rB finish_treer )
r r r rG match_setnoderesultsnew new_matchesfxrs
r r z RefactoringTool.refactor_tree s% 4>4?;; ) )E T4 ( ( ( (
14>>3C3CDDD 2DOO4E4EFFF GKK
.. )""$$%% / L .
L .
LI%%)E*:% e$))fk.?)NNN , J "%(--&+2H-III $Yu%5 6 6 $L $L 9U#333 %e , 3 3D 9 9 9% %dOOOO ) % % % %H%
. %5D<O3O3O $"'++d"3"3" L"'//$"@"@C " $ S 1 1 1,/NN,<,< !F !FD ,0+> %A>@(;$($7$>$>u$E$E$E$E /3gkk#**,,.G.G +6 !L !LC+.)+;+;79 #$-cN$9$9+c:J$K$K$K$K_ )""$$%% / Lb 4>4?;; * *E dD ) ) ) ) s F%%
F21F2c |sdS |D ]X}||j D ]H}| | }|r/| || }|| | |}IYdS )a Traverse an AST, applying a set of fixers to each node.
This is a helper method for refactor_tree().
Args:
fixers: a list of fixer instances.
traversal: a generator that yields AST nodes.
Returns:
None
N)r, r r r )r r traversalr" rG r# r$ s r r zRefactoringTool.traverse_by s F # #D *
#
# ++d++ # //$88C S))) "
# # #r c ^ | j | | | | d }|dS ||k }| |||| |r | d| | j sdS |r| |||| dS | d| dS )zR
Called when a file has been refactored and there may be changes.
Nr zNo changes to %szNot writing changes to %s)r r r r r r~
write_file)r r r r r r r s r r z RefactoringTool.processed_file s
(### //99!<H H$ (Hh>>> NN-x 8 8 8-
B OOHh( C C C C C NN6 A A A A Ar c t j |d|d }n/# t $ r"}| d|| Y d}~dS d}~ww xY w|5 | | n.# t $ r!}| d|| Y d}~nd}~ww xY wddd n # 1 swxY w Y | d| d| _ dS ) zWrites a string to a file.
It first shows a unified diff between the old text and the new text, and
then rewrites the file; the latter is only done if the write option is
set.
wr r zCan't create %s: %sNzCan't write %s: %szWrote changes to %sT)rd r r r r r r )r r r r r fpr s r r* zRefactoringTool.write_file sX 32FFFBB NN0(C @ @ @ FFFFF D D
D""""
D
D
D 3XsCCCCCCCC
D D D D D D D D D D D D D D D D
,h777
sP
AAA B A$#B $
B.B
B
BB B"%B"z>>> z... c g }d}d}d}d}| d D ])}|dz
}| | j rW|+| | |||| |}|g}| | j } |d| }|V| || j z s#||| j z dz k r| | |+| | |||| d}d}| | +|+| | |||| d
| S )a Refactors a docstring, looking for doctests.
This returns a modified version of the input string. It looks
for doctests, which start with a ">>>" prompt, and may be
continued with "..." prompts, as long as the "..." is indented
the same as the ">>>".
(Unfortunately we can't use the doctest module's parser,
since, like most parsers, it is not geared towards preserving
the original source.)
Nr Tkeependsr r r )
splitlineslstripr PS1rB refactor_doctestfindPS2rstripr r )
r r r resultblock block_linenoindentlinenolineis
r r z"RefactoringTool.refactor_docstring s $$d$33 $ $D aKF{{}}''11
$$MM$"7"7|8>#J #J K K K % IIdh'' bqb$??6DH#455 %6DHOO$5$55<<< T""""$MM$"7"7|8>#J #J K K K
d#### MM$//|06B B
C
C
Cwwv r c || }n# t $ r} j t j r.|D ]+} d| d , d|||j j
| |cY d}~S d}~ww xY w || rt |
d }|d|dz
||dz
d }} |d d s|dxx dz
cc< j z | d z g}|r| fd
|D z
}|S ) zRefactors one doctest.
A doctest is given as a block of lines, the first of which starts
with ">>>" (possibly indented), while the remaining lines start
with "..." (identically indented).
z
Source: %sr z+Can't parse docstring in %s line %s: %s: %sNTr/ r r r c * g | ]}j z |z S r& )r6 )rO r= r; r s r rR z4RefactoringTool.refactor_doctest.<locals>.<listcomp>^ s% CCCt&48+d2CCCr ) parse_blockr3 r isEnabledForr DEBUGr r7 r r r# r r r1 endswithr3 pop)
r r9 r< r; r r r r= r$ clippeds
` ` r r4 z RefactoringTool.refactor_doctestD s ##E66::DD {''
66
D ! D DDNN<T1B1BCCCC NNH#VS]-CS
J
J
JLLLLLL
dH - - Dd))&&&55C yqy>3vaxyy>SGr7##D))
B4dh&34E
DCCCCCsCCCC s
B'A6B" B'"B'c 6 | j rd}nd}| j s| d| n5| d| | j D ]}| | | j r4| d | j D ]}| | | j rut | j dk r| d n(| dt | j | j D ]\ }}} | j |g|R i | d S d S ) Nwerez
need to bezNo files %s modified.zFiles that %s modified:z$Warnings/messages while refactoring:r zThere was 1 error:zThere were %d errors:)r r r r r r )r rH filemessager r r s r summarizezRefactoringTool.summarizea sk : DD Dz ' 4d ; ; ; ; 6 = = =
'
' &&&& > * C D D D >
*
* )))) ; 54; 1$$ !56666 !8#dk:J:JKKK#';
5
5T4 4t444t4444
5 5
5
5r c | j | ||| }t |_ |S )zParses a block into a tree.
This is necessary to get correct line number / offset information
in the parser diagnostics and embedded into the parse tree.
)r parse_tokens wrap_toksrg r )r r9 r< r; r s r rA zRefactoringTool.parse_blockx s: {''uff(M(MNN ({{ r c # K t j | || j }|D ]+\ }}\ }}\ } }
} ||dz
z
}| |dz
z
} ||||f| |
f| fV ,dS )z;Wraps a tokenize stream to systematically modify start/end.r N)r rc gen_lines__next__) r r9 r< r; rA r, ru line0col0line1col1 line_texts r rN zRefactoringTool.wrap_toks s )$..*G*G*PQQDJ G G @D% %
y VaZ E VaZ E t}udmYF F F F F G Gr c # K || j z }|| j z }|}|D ]h}| | r|t | d V n5|| dz k rdV nt d|d| |}i dV )zGenerates lines as expected by tokenize from a list of lines.
This strips the first len(indent + self.PS1) characters off each line.
Nr zline=z , prefix=Tr )r3 r6 r r r7 AssertionError)r r9 r; prefix1prefix2prefixr= s r rP zRefactoringTool.gen_lines s
48#48# Dv&&
L3v;;<<((((( 4///
$nTTT66%JKKK FF HHH r r )FF)F)NFNrU ) r# r$ r% r r r r r r r r r r r r r r r r r r r* r3 r6 r r4 rK rA rN rP r& r r r{ r{ s +0).279 9 LK3L 3L 3L 3Ln&5 &5 &5P
F F F FL L L L(& & & = = = =. 66 6 6 6 M M M ^# # #. GL $B B B B* * C
C) ) ) V :5 5 5. G G G r r{ c e Zd ZdS )MultiprocessingUnsupportedNr" r& r r r] r] r' r r] c B e Zd Z fdZ d fd Z fdZ fdZ xZS )MultiprocessRefactoringToolc d t t | j |i | d | _ d | _ d S rU )superr_ r queue output_lockr r kwargsr s r r z$MultiprocessRefactoringTool.__init__ s; 9)40094J6JJJ
r Fr c |dk r*t t ||| S dd ln# t $ r t
w xY w j t d _ _
fdt | D } |D ]}| t t ||| j
t | D ] } j d |D ]*}| r|
+d _ d S # j
t | D ] } j d |D ]*}| r|
+d _ w xY w)Nr r z already doing multiple processesc F g | ] } j S ))target)Process_child)rO r> multiprocessingr s r rR z8MultiprocessRefactoringTool.refactor.<locals>.<listcomp> s< 4 4 4 %,,DK,@@ 4 4 4r )ra r_ r rk ImportErrorr] rb RuntimeError
JoinableQueueLockrc rangestartr putis_alive)
r r r r
num_processes processesr6 r> rk r s
` @r r z$MultiprocessRefactoringTool.refactor s/ A 4d;;DDum- -
- - " " " " " - - -, , - : ! ABB B$2244
*//114 4 4 4 4 #M 2 24 4 4
-t 4 4 = =eU>K
M
M
M
JOO =))
%
%
t$$$$
::<< FFHHH DJJJ
JOO =))
%
%
t$$$$
::<< FFHHH DJ s : A 4AE2 2A;G-c 4 | j }|{|\ }} t t | j |i | | j n # | j w xY w| j }|yd S d S rU )rb r ra r_ r task_done)r taskr re r s r rj z"MultiprocessRefactoringTool._child s z~~ LD&
'F1488F% #% % %
$$&&&&
$$&&&&:>>##D s A A8c | j | j ||f d S t t | j |i |S rU )rb rr ra r_ r rd s r r z)MultiprocessRefactoringTool.refactor_file sV : ! JNND&> * * * * *I54d;;I! ! !
!r )FFr )r# r$ r% r r rj r
__classcell__)r s @r r_ r_ s
:? : $ $ $ $ $! ! ! ! ! ! ! ! !r r_ )T)#ry
__author__rd r r r r r r9 itertoolsr pgen2r r r
fixer_utilr r r r
r r r r3 r! r/ rJ rS rW rv rx objectr{ r] r_ r& r r <module>r s 3
+ * * * * * * * * * ! ! ! ! ! ! C C C8 2@ @ @ % % % P' ' ' ' ' ' ' 'F F F F Ff F F FR 4! 4! 4! 4! 4!/ 4! 4! 4! 4! 4!r