Skip to content

Instantly share code, notes, and snippets.

@wowkin2
Last active November 20, 2020 09:14
Show Gist options
  • Save wowkin2/6a437fb4f407709a2c136565cd9f7c91 to your computer and use it in GitHub Desktop.
Save wowkin2/6a437fb4f407709a2c136565cd9f7c91 to your computer and use it in GitHub Desktop.
Convert line ending from Windows CRLF to Unix LF (Ubuntu, Linux)
import os
CRLF = b'\r\n'
LF = b'\n'
allowed_extensions = [
'gitignore', 'Procfile', 'Docker',
'txt', 'md', 'ini', 'json', 'yaml', # NOTE: do not add csv, they can be huge
'js', 'css', 'scss', 'html', 'htm', 'svg',
'py', 'ipynb', 'pylintrc',
'php',
]
def get_git_ignore():
with open('.gitignore', 'r') as f:
result = f.read()
result = result.splitlines()
result.extend([
'.git/',
])
return result
def is_ignorable(path, ignore_list):
if '/' in path:
base_folder = path[:path.index('/')]
else:
base_folder = path
# TODO: properly check for root and non-root folders
# current_folder = base_folder + '/'
# root_folder = '/' + base_folder + '/'
if base_folder + '/' in ignore_list or '/' + base_folder + '/' in ignore_list:
return True
return False
def get_files_list(path, ignore_list=None):
result = []
for root, folders, files in os.walk(path):
relative_path = root[len(path)+1:]
if is_ignorable(relative_path, ignore_list):
continue
for name in files:
_, extension = name.rsplit('.', maxsplit=1) if '.' in name else (None, name)
if extension in allowed_extensions:
full_path = os.path.join(root, name)
if not is_ignorable(name, ignore_list):
result.append(full_path)
return result
def process_file(file_path):
print('Processing file: ', file_path)
with open(file_path, 'rb') as f:
content = f.read()
content = content.replace(CRLF, LF)
with open(file_path, 'wb') as f:
f.write(content)
def main(path):
print('Start processing path: ', path)
git_ignore = get_git_ignore()
list_file_path = get_files_list(path, git_ignore)
for file_path in list_file_path:
process_file(file_path)
if __name__ == '__main__':
args_path = os.path.dirname(os.path.realpath(__file__)) # TODO: read from args
main(args_path)
@wowkin2
Copy link
Author

wowkin2 commented Nov 17, 2020

Also rescan files with Git:
git add -uv

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment