Created
September 13, 2017 03:31
-
-
Save BroHui/aca2b8e6e6bdf3cb4af4b246c9837fa3 to your computer and use it in GitHub Desktop.
Remove comments and docstrings from a python fille.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" Strip comments and docstrings from a file. | |
""" | |
import sys, token, tokenize | |
def do_file(fname): | |
""" Run on just one file. | |
""" | |
source = open(fname) | |
mod = open(fname + ",strip", "w") | |
prev_toktype = token.INDENT | |
first_line = None | |
last_lineno = -1 | |
last_col = 0 | |
tokgen = tokenize.generate_tokens(source.readline) | |
for toktype, ttext, (slineno, scol), (elineno, ecol), ltext in tokgen: | |
if 0: # Change to if 1 to see the tokens fly by. | |
print("%10s %-14s %-20r %r" % ( | |
tokenize.tok_name.get(toktype, toktype), | |
"%d.%d-%d.%d" % (slineno, scol, elineno, ecol), | |
ttext, ltext | |
)) | |
if slineno > last_lineno: | |
last_col = 0 | |
if scol > last_col: | |
mod.write(" " * (scol - last_col)) | |
if toktype == token.STRING and prev_toktype == token.INDENT: | |
# Docstring | |
mod.write("#--") | |
elif toktype == tokenize.COMMENT: | |
# Comment | |
mod.write("##\n") | |
else: | |
mod.write(ttext) | |
prev_toktype = toktype | |
last_col = ecol | |
last_lineno = elineno | |
if __name__ == '__main__': | |
do_file(sys.argv[1]) |
There is a bug with the code (having to do with the logic prev_toktype == token.INDENT)
If there is a docstring (1 line or multiline) that begins at column 1 (with no preceeding spaces/tabs), it is not stripped out.
So an input of the form
""" string 1 """
""" string 2 """
In this case string 1 is not stripped out
changing
if toktype == token.STRING and prev_toktype == token.INDENT:
to
if toktype == token.STRING and (prev_toktype == token.INDENT or prev_toktype == token.NEWLINE):
does the job.
this will not generate legal code if a method has nothing but a doc string
you can check python/lib/codecs.py for example
credits: Ned Batchelder
https://stackoverflow.com/a/1769577/558008
check also the comments to his answer
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Single line of code
sed 's/#.*$//g' your_filename_here.py