Model: llama3.2:1b
Ran this against https://github.com/simonw/djp
files-to-prompt **/*.py -c | llm -m llama3.2:1b --system 'describe this code'
Full context - 3,726 tokens for the OpenAI tokenizer, not sure exactly how many for Llama tokenizer
<documents>
<document index="1">
<source>djp/__init__.py</source>
<document_content>
from .hookspecs import hookimpl
from . import hookspecs
import itertools
import os
import pathlib
from pluggy import PluginManager
import sys
from typing import List
import types
pm = PluginManager("djp")
pm.add_hookspecs(hookspecs)
pm.load_setuptools_entrypoints("djp")
def _module_from_path(path, name):
# Adapted from http://sayspy.blogspot.com/2011/07/how-to-import-module-from-just-file.html
mod = types.ModuleType(name)
mod.__file__ = path
with open(path, "r") as file:
code = compile(file.read(), path, "exec", dont_inherit=True)
exec(code, mod.__dict__)
return mod
plugins_dir = os.environ.get("DJP_PLUGINS_DIR")
if plugins_dir:
for filepath in pathlib.Path(plugins_dir).glob("*.py"):
mod = _module_from_path(str(filepath), name=filepath.stem)
try:
pm.register(mod)
except ValueError as ex:
print(ex, file=sys.stderr)
# Plugin already registered
pass
class Before:
def __init__(self, item: str):
self.item = item
class After:
def __init__(self, item: str):
self.item = item
class Position:
def __init__(self, item: str, before=None, after=None):
assert not (before and after), "Cannot specify both before and after"
self.item = item
self.before = before
self.after = after
def installed_apps() -> List[str]:
return ["djp"] + list(itertools.chain(*pm.hook.installed_apps()))
def middleware(current_middleware: List[str]):
before = []
after = []
default = []
position_items = []
for batch in pm.hook.middleware():
for item in batch:
if isinstance(item, Before):
before.append(item.item)
elif isinstance(item, After):
after.append(item.item)
elif isinstance(item, Position):
position_items.append(item)
elif isinstance(item, str):
default.append(item)
else:
raise ValueError(f"Invalid item in middleware hook: {item}")
combined = before + to_list(current_middleware) + default + after
# Handle Position items
for item in position_items:
if item.before:
try:
idx = combined.index(item.before)
combined.insert(idx, item.item)
except ValueError:
raise ValueError(f"Cannot find item to insert before: {item.before}")
elif item.after:
try:
idx = combined.index(item.after)
combined.insert(idx + 1, item.item)
except ValueError:
raise ValueError(f"Cannot find item to insert after: {item.after}")
print("")
print("combined", combined)
print("")
return combined
def urlpatterns():
return list(itertools.chain(*pm.hook.urlpatterns()))
def settings(current_settings):
# First wrap INSTALLED_APPS
installed_apps_ = to_list(current_settings["INSTALLED_APPS"])
installed_apps_ += installed_apps()
current_settings["INSTALLED_APPS"] = installed_apps_
# Now MIDDLEWARE
current_settings["MIDDLEWARE"] = middleware(current_settings["MIDDLEWARE"])
# Now apply any other settings() hooks
pm.hook.settings(current_settings=current_settings)
def get_plugins():
plugins = []
plugin_to_distinfo = dict(pm.list_plugin_distinfo())
for plugin in pm.get_plugins():
plugin_info = {
"name": plugin.__name__,
"hooks": [h.name for h in pm.get_hookcallers(plugin)],
}
distinfo = plugin_to_distinfo.get(plugin)
if distinfo:
plugin_info["version"] = distinfo.version
plugin_info["name"] = (
getattr(distinfo, "name", None) or distinfo.project_name
)
plugins.append(plugin_info)
return plugins
def to_list(tuple_or_list):
if isinstance(tuple_or_list, tuple):
return list(tuple_or_list)
return tuple_or_list
</document_content>
</document>
<document index="2">
<source>djp/hookspecs.py</source>
<document_content>
from pluggy import HookimplMarker
from pluggy import HookspecMarker
hookspec = HookspecMarker("djp")
hookimpl = HookimplMarker("djp")
@hookspec
def installed_apps():
"""Return a list of Django app strings to be added to INSTALLED_APPS"""
@hookspec
def middleware():
"""
Return a list of Django middleware class strings to be added to MIDDLEWARE.
Optionally wrap with djp.Before() or djp.After() to specify ordering,
or wrap with djp.Position(name, before=other_name) to insert before another
or djp.Position(name, after=other_name) to insert after another.
"""
@hookspec
def urlpatterns():
"""Return a list of url patterns to be added to urlpatterns"""
@hookspec
def settings(current_settings):
"""Modify current_settings in place to finish configuring settings.py"""
</document_content>
</document>
<document index="3">
<source>djp/management/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="4">
<source>djp/management/commands/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="5">
<source>djp/management/commands/showplugins.py</source>
<document_content>
from django.core.management.base import BaseCommand
from djp import get_plugins
import json
class Command(BaseCommand):
help = "Show installed plugins"
def handle(self, *args, **options):
plugins = get_plugins()
print(json.dumps(plugins, indent=2))
</document_content>
</document>
<document index="6">
<source>docs/conf.py</source>
<document_content>
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import datetime
from subprocess import PIPE, Popen
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ["myst_parser", "sphinx_copybutton"]
myst_enable_extensions = ["colon_fence"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = ".rst"
# The master toctree document.
master_doc = "index"
# General information about the project.
project = "DJP"
author = "Simon Willison"
copyright = "{}, {}".format(datetime.date.today().year, author)
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True)
git_version = pipe.stdout.read().decode("utf8")
if git_version:
version = git_version.rsplit("-", 1)[0]
release = git_version
else:
version = ""
release = ""
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = "en"
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = "furo"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
html_theme_options = {}
html_title = project
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = "project-doc"
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(
master_doc,
"project.tex",
project,
author,
"manual",
)
]
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(
master_doc,
"project",
project,
[author],
1,
)
]
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(
master_doc,
"project",
project,
author,
"project",
"Documentation for {}".format(project),
"Miscellaneous",
)
]
</document_content>
</document>
<document index="7">
<source>tests/plugins/installed_apps.py</source>
<document_content>
import djp
@djp.hookimpl
def installed_apps():
return ["tests.test_project.app1"]
</document_content>
</document>
<document index="8">
<source>tests/plugins/middleware.py</source>
<document_content>
import djp
@djp.hookimpl(specname="middleware", tryfirst=True)
def middleware1():
return [
"tests.test_project.middleware.Middleware",
"tests.test_project.middleware.Middleware2",
"tests.test_project.middleware.Middleware3",
djp.Before("tests.test_project.middleware.MiddlewareBefore"),
djp.After("tests.test_project.middleware.MiddlewareAfter"),
]
@djp.hookimpl(specname="middleware")
def middleware2():
return [
djp.Position(
"tests.test_project.middleware.Middleware4",
before="tests.test_project.middleware.Middleware2",
),
djp.Position(
"tests.test_project.middleware.Middleware5",
before="tests.test_project.middleware.Middleware3",
),
]
</document_content>
</document>
<document index="9">
<source>tests/plugins/settings.py</source>
<document_content>
import djp
@djp.hookimpl
def settings(current_settings):
current_settings["FROM_PLUGIN"] = "x"
</document_content>
</document>
<document index="10">
<source>tests/plugins/urlpatterns.py</source>
<document_content>
from django.urls import path
from django.http import HttpResponse
import djp
@djp.hookimpl
def urlpatterns():
return [path("from-plugin/", lambda request: HttpResponse("Hello from a plugin"))]
</document_content>
</document>
<document index="11">
<source>tests/test_django_plugins.py</source>
<document_content>
from django.conf import settings
from django.test.client import Client
def test_middleware_order():
assert settings.MIDDLEWARE == [
"tests.test_project.middleware.MiddlewareBefore",
"tests.test_project.middleware.Middleware",
"tests.test_project.middleware.Middleware4",
"tests.test_project.middleware.Middleware2",
"tests.test_project.middleware.Middleware5",
"tests.test_project.middleware.Middleware3",
"tests.test_project.middleware.MiddlewareAfter",
]
def test_middleware():
response = Client().get("/")
assert response["X-DJP-Middleware-After"] == "MiddlewareAfter"
assert response["X-DJP-Middleware"] == "Middleware"
assert response["X-DJP-Middleware-Before"] == "MiddlewareBefore"
request = response._request
assert hasattr(request, "_notes")
assert request._notes == [
"MiddlewareAfter",
"Middleware3",
"Middleware5",
"Middleware2",
"Middleware4",
"Middleware",
"MiddlewareBefore",
]
def test_urlpatterns():
response = Client().get("/from-plugin/")
assert response.content == b"Hello from a plugin"
def test_settings():
assert settings.FROM_PLUGIN == "x"
def test_installed_apps():
assert "tests.test_project.app1" in settings.INSTALLED_APPS
</document_content>
</document>
<document index="12">
<source>tests/test_project/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="13">
<source>tests/test_project/app1/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="14">
<source>tests/test_project/middleware.py</source>
<document_content>
def request_note(request, response, note):
if not hasattr(request, "_notes"):
request._notes = []
request._notes.append(note)
response._request = request
class Middleware:
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
response = self.get_response(request)
response["X-DJP-Middleware"] = "Middleware"
request_note(request, response, "Middleware")
return response
class Middleware2:
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
response = self.get_response(request)
request_note(request, response, self.__class__.__name__)
return response
class Middleware3(Middleware2):
pass
class Middleware4(Middleware2):
pass
class Middleware5(Middleware2):
pass
class MiddlewareBefore:
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
response = self.get_response(request)
response["X-DJP-Middleware-Before"] = "MiddlewareBefore"
request_note(request, response, "MiddlewareBefore")
return response
class MiddlewareAfter:
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
response = self.get_response(request)
response["X-DJP-Middleware-After"] = "MiddlewareAfter"
request_note(request, response, "MiddlewareAfter")
return response
</document_content>
</document>
<document index="15">
<source>tests/test_project/settings.py</source>
<document_content>
import djp
SECRET_KEY = "django-insecure-test-key"
DEBUG = True
ALLOWED_HOSTS = ["*"]
INSTALLED_APPS = (
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
)
MIDDLEWARE = tuple()
ROOT_URLCONF = "tests.test_project.urls"
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": ":memory:",
}
}
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"APP_DIRS": True,
}
]
USE_TZ = True
djp.settings(globals())
</document_content>
</document>
<document index="16">
<source>tests/test_project/urls.py</source>
<document_content>
from django.http import HttpResponse
from django.urls import path
import djp
urlpatterns = [
path("", lambda request: HttpResponse("Hello world"))
] + djp.urlpatterns()
</document_content>
</document>
</documents>
describe this code
This is a Django project's test setup file, specifically the tests/test_django_plugins.py file. Here's a breakdown of what this file does:
Defining test functions
The first section defines several test functions using the @djp.hookimpl decorator from the djp library. These hook implementations allow you to intercept and manipulate Django's behavior.
test_middleware_order: This function checks that the middleware order is correct by comparing theMIDDLEWAREsetting with a predefined list.test_middleware: This function tests various aspects of middleware:- It retrieves the response from the URL
/from-plugin/using theClientobject, which simulates a request to this view. - It checks that certain values are present in the response:
X-DJP-Middleware-AfterX-DJP-MiddlewareX-DJP-Middleware-Before
- It retrieves the response from the URL
test_urlpatterns: This function tests that the URL patterns for views work as expected.test_settings: This function checks that certain values are present in the settings:FROM_PLUGINis set to"x"
test_installed_apps: This function checks that an app named"tests.test_project.app1"is installed.
Defining test classes
The next section defines a simple test class using the Django built-in TestCase class:
__init__.py: This file contains some basic setup for the test project.app1/__init__.py: This file defines an app named"tests.test_project.app1"and imports it into the current namespace.
Defining middleware classes
The following section defines two custom middleware classes that will be used in the tests:
Middlewareclass: This middleware allows you to set a "middleware" key on the response object.Middleware2class: This middleware is similar toMiddleware, but it adds some additional functionality.
Defining test hooks
The final section defines three custom hook implementations that will be used in the tests:
request_note: This function allows you to set a note on the request object, which can be accessed later.Middleware3andMiddleware4classes: These are simple middleware classes that add additional information to the response object.
Overall, this test setup file is designed to verify various aspects of Django's behavior in specific scenarios.