Skip to content

Instantly share code, notes, and snippets.

@simonw

simonw/warnings.md Secret

Created April 8, 2024 06:02
Show Gist options
  • Save simonw/9b83f42a1b87d3fcb3b4b8e6f482af38 to your computer and use it in GitHub Desktop.
Save simonw/9b83f42a1b87d3fcb3b4b8e6f482af38 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
warnings.md

2024-04-08T05:47:36 conversation: 01hty405p89t505cf5kwgnm79z

Model: claude-3-opus-20240229

Prompt:

files_to_prompt/cli.py
---
import os
import click
from pathlib import Path
from fnmatch import fnmatch


def should_ignore(path, gitignore_rules):
    for rule in gitignore_rules:
        if fnmatch(os.path.basename(path), rule):
            return True
        if os.path.isdir(path) and fnmatch(os.path.basename(path) + "/", rule):
            return True
    return False


def read_gitignore(path):
    gitignore_path = os.path.join(path, ".gitignore")
    if os.path.isfile(gitignore_path):
        with open(gitignore_path, "r") as f:
            return [
                line.strip() for line in f if line.strip() and not line.startswith("#")
            ]
    return []


def process_path(
    path, include_hidden, ignore_gitignore, gitignore_rules, ignore_patterns
):
    if os.path.isfile(path):
        with open(path, "r") as f:
            file_contents = f.read()
        click.echo(path)
        click.echo("---")
        click.echo(file_contents)
        click.echo()
        click.echo("---")
    elif os.path.isdir(path):
        for root, dirs, files in os.walk(path):
            if not include_hidden:
                dirs[:] = [d for d in dirs if not d.startswith(".")]
                files = [f for f in files if not f.startswith(".")]

            if not ignore_gitignore:
                gitignore_rules.extend(read_gitignore(root))
                dirs[:] = [
                    d
                    for d in dirs
                    if not should_ignore(os.path.join(root, d), gitignore_rules)
                ]
                files = [
                    f
                    for f in files
                    if not should_ignore(os.path.join(root, f), gitignore_rules)
                ]

            if ignore_patterns:
                files = [
                    f
                    for f in files
                    if not any(fnmatch(f, pattern) for pattern in ignore_patterns)
                ]

            for file in files:
                file_path = os.path.join(root, file)
                with open(file_path, "r") as f:
                    file_contents = f.read()

                click.echo(file_path)
                click.echo("---")
                click.echo(file_contents)
                click.echo()
                click.echo("---")


@click.command()
@click.argument("paths", nargs=-1, type=click.Path(exists=True))
@click.option(
    "--include-hidden",
    is_flag=True,
    help="Include files and folders starting with .",
)
@click.option(
    "--ignore-gitignore",
    is_flag=True,
    help="Ignore .gitignore files and include all files",
)
@click.option(
    "ignore_patterns",
    "--ignore",
    multiple=True,
    default=[],
    help="List of patterns to ignore",
)
@click.version_option()
def cli(paths, include_hidden, ignore_gitignore, ignore_patterns):
    """
    Takes one or more paths to files or directories and outputs every file,
    recursively, each one preceded with its filename like this:

    path/to/file.py
    ----
    Contents of file.py goes here

    ---
    path/to/file2.py
    ---
    ...
    """
    gitignore_rules = []
    for path in paths:
        if not os.path.exists(path):
            raise click.BadArgumentUsage(f"Path does not exist: {path}")
        if not ignore_gitignore:
            gitignore_rules.extend(read_gitignore(os.path.dirname(path)))
        process_path(
            path, include_hidden, ignore_gitignore, gitignore_rules, ignore_patterns
        )

System:

catch unicodedecodeerror reading the file and output a click warning about the file, skipping it and moving on

Response:

Here's the updated cli.py script with the requested changes to catch UnicodeDecodeError and output a click warning about the file, skipping it and moving on:

import os
import click
from pathlib import Path
from fnmatch import fnmatch


def should_ignore(path, gitignore_rules):
    for rule in gitignore_rules:
        if fnmatch(os.path.basename(path), rule):
            return True
        if os.path.isdir(path) and fnmatch(os.path.basename(path) + "/", rule):
            return True
    return False


def read_gitignore(path):
    gitignore_path = os.path.join(path, ".gitignore")
    if os.path.isfile(gitignore_path):
        with open(gitignore_path, "r") as f:
            return [
                line.strip() for line in f if line.strip() and not line.startswith("#")
            ]
    return []


def process_path(
    path, include_hidden, ignore_gitignore, gitignore_rules, ignore_patterns
):
    if os.path.isfile(path):
        try:
            with open(path, "r") as f:
                file_contents = f.read()
            click.echo(path)
            click.echo("---")
            click.echo(file_contents)
            click.echo()
            click.echo("---")
        except UnicodeDecodeError:
            click.echo(f"Warning: Skipping file {path} due to UnicodeDecodeError")
    elif os.path.isdir(path):
        for root, dirs, files in os.walk(path):
            if not include_hidden:
                dirs[:] = [d for d in dirs if not d.startswith(".")]
                files = [f for f in files if not f.startswith(".")]

            if not ignore_gitignore:
                gitignore_rules.extend(read_gitignore(root))
                dirs[:] = [
                    d
                    for d in dirs
                    if not should_ignore(os.path.join(root, d), gitignore_rules)
                ]
                files = [
                    f
                    for f in files
                    if not should_ignore(os.path.join(root, f), gitignore_rules)
                ]

            if ignore_patterns:
                files = [
                    f
                    for f in files
                    if not any(fnmatch(f, pattern) for pattern in ignore_patterns)
                ]

            for file in files:
                file_path = os.path.join(root, file)
                try:
                    with open(file_path, "r") as f:
                        file_contents = f.read()

                    click.echo(file_path)
                    click.echo("---")
                    click.echo(file_contents)
                    click.echo()
                    click.echo("---")
                except UnicodeDecodeError:
                    click.echo(f"Warning: Skipping file {file_path} due to UnicodeDecodeError")


@click.command()
@click.argument("paths", nargs=-1, type=click.Path(exists=True))
@click.option(
    "--include-hidden",
    is_flag=True,
    help="Include files and folders starting with .",
)
@click.option(
    "--ignore-gitignore",
    is_flag=True,
    help="Ignore .gitignore files and include all files",
)
@click.option(
    "ignore_patterns",
    "--ignore",
    multiple=True,
    default=[],
    help="List of patterns to ignore",
)
@click.version_option()
def cli(paths, include_hidden, ignore_gitignore, ignore_patterns):
    """
    Takes one or more paths to files or directories and outputs every file,
    recursively, each one preceded with its filename like this:

    path/to/file.py
    ----
    Contents of file.py goes here

    ---
    path/to/file2.py
    ---
    ...
    """
    gitignore_rules = []
    for path in paths:
        if not os.path.exists(path):
            raise click.BadArgumentUsage(f"Path does not exist: {path}")
        if not ignore_gitignore:
            gitignore_rules.extend(read_gitignore(os.path.dirname(path)))
        process_path(
            path, include_hidden, ignore_gitignore, gitignore_rules, ignore_patterns
        )

The main changes are:

  1. In the process_path function, I added a try-except block around the file reading and printing code. If a UnicodeDecodeError is caught while reading the file, it prints a warning message using click.echo and skips that file, moving on to the next one.

  2. This change is applied both when processing a single file and when processing files within a directory.

Now, if the script encounters a file that cannot be decoded with the default encoding, it will output a warning message like this:

Warning: Skipping file path/to/file.txt due to UnicodeDecodeError

The script will then continue processing the remaining files without interruption.

2024-04-08T05:50:38 conversation: 01hty405p89t505cf5kwgnm79z

Model: claude-3-opus-20240229

Prompt:

remember to use err=True on those click echo lines

Response:

You're right, my apologies for missing that. Here's the updated cli.py script with err=True added to the click.echo lines for warning messages:

import os
import click
from pathlib import Path
from fnmatch import fnmatch


def should_ignore(path, gitignore_rules):
    for rule in gitignore_rules:
        if fnmatch(os.path.basename(path), rule):
            return True
        if os.path.isdir(path) and fnmatch(os.path.basename(path) + "/", rule):
            return True
    return False


def read_gitignore(path):
    gitignore_path = os.path.join(path, ".gitignore")
    if os.path.isfile(gitignore_path):
        with open(gitignore_path, "r") as f:
            return [
                line.strip() for line in f if line.strip() and not line.startswith("#")
            ]
    return []


def process_path(
    path, include_hidden, ignore_gitignore, gitignore_rules, ignore_patterns
):
    if os.path.isfile(path):
        try:
            with open(path, "r") as f:
                file_contents = f.read()
            click.echo(path)
            click.echo("---")
            click.echo(file_contents)
            click.echo()
            click.echo("---")
        except UnicodeDecodeError:
            click.echo(f"Warning: Skipping file {path} due to UnicodeDecodeError", err=True)
    elif os.path.isdir(path):
        for root, dirs, files in os.walk(path):
            if not include_hidden:
                dirs[:] = [d for d in dirs if not d.startswith(".")]
                files = [f for f in files if not f.startswith(".")]

            if not ignore_gitignore:
                gitignore_rules.extend(read_gitignore(root))
                dirs[:] = [
                    d
                    for d in dirs
                    if not should_ignore(os.path.join(root, d), gitignore_rules)
                ]
                files = [
                    f
                    for f in files
                    if not should_ignore(os.path.join(root, f), gitignore_rules)
                ]

            if ignore_patterns:
                files = [
                    f
                    for f in files
                    if not any(fnmatch(f, pattern) for pattern in ignore_patterns)
                ]

            for file in files:
                file_path = os.path.join(root, file)
                try:
                    with open(file_path, "r") as f:
                        file_contents = f.read()

                    click.echo(file_path)
                    click.echo("---")
                    click.echo(file_contents)
                    click.echo()
                    click.echo("---")
                except UnicodeDecodeError:
                    click.echo(f"Warning: Skipping file {file_path} due to UnicodeDecodeError", err=True)


@click.command()
@click.argument("paths", nargs=-1, type=click.Path(exists=True))
@click.option(
    "--include-hidden",
    is_flag=True,
    help="Include files and folders starting with .",
)
@click.option(
    "--ignore-gitignore",
    is_flag=True,
    help="Ignore .gitignore files and include all files",
)
@click.option(
    "ignore_patterns",
    "--ignore",
    multiple=True,
    default=[],
    help="List of patterns to ignore",
)
@click.version_option()
def cli(paths, include_hidden, ignore_gitignore, ignore_patterns):
    """
    Takes one or more paths to files or directories and outputs every file,
    recursively, each one preceded with its filename like this:

    path/to/file.py
    ----
    Contents of file.py goes here

    ---
    path/to/file2.py
    ---
    ...
    """
    gitignore_rules = []
    for path in paths:
        if not os.path.exists(path):
            raise click.BadArgumentUsage(f"Path does not exist: {path}")
        if not ignore_gitignore:
            gitignore_rules.extend(read_gitignore(os.path.dirname(path)))
        process_path(
            path, include_hidden, ignore_gitignore, gitignore_rules, ignore_patterns
        )

The warning messages are now printed using click.echo with the err=True parameter. This ensures that the warning messages are sent to the standard error stream (stderr) instead of the standard output stream (stdout).

Thank you for pointing out the improvement!

2024-04-08T05:51:50 conversation: 01hty405p89t505cf5kwgnm79z

Model: claude-3-opus-20240229

Prompt:

How would I show those in a different color?

Response:

To display the warning messages in a different color, you can use ANSI color codes. Click provides a built-in way to style the output using the click.style function. Here's how you can modify the script to show the warning messages in red color:

import os
import click
from pathlib import Path
from fnmatch import fnmatch


def should_ignore(path, gitignore_rules):
    for rule in gitignore_rules:
        if fnmatch(os.path.basename(path), rule):
            return True
        if os.path.isdir(path) and fnmatch(os.path.basename(path) + "/", rule):
            return True
    return False


def read_gitignore(path):
    gitignore_path = os.path.join(path, ".gitignore")
    if os.path.isfile(gitignore_path):
        with open(gitignore_path, "r") as f:
            return [
                line.strip() for line in f if line.strip() and not line.startswith("#")
            ]
    return []


def process_path(
    path, include_hidden, ignore_gitignore, gitignore_rules, ignore_patterns
):
    if os.path.isfile(path):
        try:
            with open(path, "r") as f:
                file_contents = f.read()
            click.echo(path)
            click.echo("---")
            click.echo(file_contents)
            click.echo()
            click.echo("---")
        except UnicodeDecodeError:
            warning_message = f"Warning: Skipping file {path} due to UnicodeDecodeError"
            click.echo(click.style(warning_message, fg='red'), err=True)
    elif os.path.isdir(path):
        for root, dirs, files in os.walk(path):
            if not include_hidden:
                dirs[:] = [d for d in dirs if not d.startswith(".")]
                files = [f for f in files if not f.startswith(".")]

            if not ignore_gitignore:
                gitignore_rules.extend(read_gitignore(root))
                dirs[:] = [
                    d
                    for d in dirs
                    if not should_ignore(os.path.join(root, d), gitignore_rules)
                ]
                files = [
                    f
                    for f in files
                    if not should_ignore(os.path.join(root, f), gitignore_rules)
                ]

            if ignore_patterns:
                files = [
                    f
                    for f in files
                    if not any(fnmatch(f, pattern) for pattern in ignore_patterns)
                ]

            for file in files:
                file_path = os.path.join(root, file)
                try:
                    with open(file_path, "r") as f:
                        file_contents = f.read()

                    click.echo(file_path)
                    click.echo("---")
                    click.echo(file_contents)
                    click.echo()
                    click.echo("---")
                except UnicodeDecodeError:
                    warning_message = f"Warning: Skipping file {file_path} due to UnicodeDecodeError"
                    click.echo(click.style(warning_message, fg='red'), err=True)


@click.command()
@click.argument("paths", nargs=-1, type=click.Path(exists=True))
@click.option(
    "--include-hidden",
    is_flag=True,
    help="Include files and folders starting with .",
)
@click.option(
    "--ignore-gitignore",
    is_flag=True,
    help="Ignore .gitignore files and include all files",
)
@click.option(
    "ignore_patterns",
    "--ignore",
    multiple=True,
    default=[],
    help="List of patterns to ignore",
)
@click.version_option()
def cli(paths, include_hidden, ignore_gitignore, ignore_patterns):
    """
    Takes one or more paths to files or directories and outputs every file,
    recursively, each one preceded with its filename like this:

    path/to/file.py
    ----
    Contents of file.py goes here

    ---
    path/to/file2.py
    ---
    ...
    """
    gitignore_rules = []
    for path in paths:
        if not os.path.exists(path):
            raise click.BadArgumentUsage(f"Path does not exist: {path}")
        if not ignore_gitignore:
            gitignore_rules.extend(read_gitignore(os.path.dirname(path)))
        process_path(
            path, include_hidden, ignore_gitignore, gitignore_rules, ignore_patterns
        )

The changes made:

  1. The warning messages are now stored in a variable called warning_message.

  2. The click.style function is used to style the warning_message. The fg parameter is set to 'red' to set the foreground color to red.

  3. The styled warning message is then passed to click.echo with err=True to display it in the standard error stream.

Now, when a UnicodeDecodeError occurs, the warning message will be displayed in red color, making it more noticeable and distinguishable from the regular output.

You can choose different color names supported by Click, such as 'green', 'blue', 'yellow', etc., or use ANSI color codes for more color options.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment