Model: gemini-2.5-pro-exp-03-25
files-to-prompt . -e py -c
<documents>
<document index="1">
<source>./manage.py</source>
<document_content>
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
</document_content>
</document>
<document index="2">
<source>./cloudflareips/ip_ranges.py</source>
<document_content>
from __future__ import unicode_literals
import ipaddress
IP_RANGES = [
ipaddress.ip_network("103.21.244.0/22"),
ipaddress.ip_network("103.22.200.0/22"),
ipaddress.ip_network("103.31.4.0/22"),
ipaddress.ip_network("104.16.0.0/12"),
ipaddress.ip_network("108.162.192.0/18"),
ipaddress.ip_network("131.0.72.0/22"),
ipaddress.ip_network("141.101.64.0/18"),
ipaddress.ip_network("162.158.0.0/15"),
ipaddress.ip_network("172.64.0.0/13"),
ipaddress.ip_network("173.245.48.0/20"),
ipaddress.ip_network("188.114.96.0/20"),
ipaddress.ip_network("190.93.240.0/20"),
ipaddress.ip_network("197.234.240.0/22"),
ipaddress.ip_network("198.41.128.0/17"),
ipaddress.ip_network("2400:cb00::/32"),
ipaddress.ip_network("2405:8100::/32"),
ipaddress.ip_network("2405:b500::/32"),
ipaddress.ip_network("2606:4700::/32"),
ipaddress.ip_network("2803:f800::/32"),
ipaddress.ip_network("2c0f:f248::/32"),
ipaddress.ip_network("2a06:98c0::/29"),
]
</document_content>
</document>
<document index="3">
<source>./config/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="4">
<source>./config/hosts.py</source>
<document_content>
from django.conf import settings
from django_hosts import patterns, host
host_patterns = patterns(
"",
host(r"www", settings.ROOT_URLCONF, name="www"),
host(r"2003", "config.urls_2003", name="urls_2003"),
)
</document_content>
</document>
<document index="5">
<source>./config/settings.py</source>
<document_content>
import os
import dj_database_url
import djp
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get("DJANGO_SECRET") or "dev-secret-s(p7%ue-l6r^&@y63p*ix*1"
# CSRF trusted origins
CSRF_TRUSTED_ORIGINS = []
if os.environ.get("CSRF_TRUSTED_ORIGINS"):
CSRF_TRUSTED_ORIGINS = os.environ["CSRF_TRUSTED_ORIGINS"].split(",")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = bool(os.environ.get("DJANGO_DEBUG"))
INTERNAL_IPS = ("127.0.0.1",)
STAGING = bool(os.environ.get("STAGING"))
# Django SQL Dashboard
DASHBOARD_ROW_LIMIT = 200
# Cloudflare details
CLOUDFLARE_EMAIL = os.environ.get("CLOUDFLARE_EMAIL", "")
CLOUDFLARE_TOKEN = os.environ.get("CLOUDFLARE_TOKEN", "")
CLOUDFLARE_ZONE_ID = os.environ.get("CLOUDFLARE_ZONE_ID", "")
# https://github.com/simonw/simonwillisonblog/issues/498
SECURE_CROSS_ORIGIN_OPENER_POLICY = False
# SESSION_COOKIE_DOMAIN
if os.environ.get("SESSION_COOKIE_DOMAIN"):
SESSION_COOKIE_DOMAIN = os.environ["SESSION_COOKIE_DOMAIN"]
if os.environ.get("SESSION_COOKIE_SECURE"):
SESSION_COOKIE_SECURE = True
# Application definition
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"django_sql_dashboard",
"django.contrib.humanize",
"django_hosts",
"blog",
"redirects",
"feedstats",
"django_http_debug",
]
MIDDLEWARE = [
"django_hosts.middleware.HostsRequestMiddleware",
"redirects.middleware.redirect_middleware",
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django_http_debug.middleware.DebugMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"blog.middleware.AmpersandRedirectMiddleware",
"django_hosts.middleware.HostsResponseMiddleware",
# "django.middleware.clickjacking.XFrameOptionsMiddleware",
]
if DEBUG:
try:
import debug_toolbar
MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"]
INSTALLED_APPS += ["debug_toolbar"]
except ImportError:
pass
MIDDLEWARE += ["whitenoise.middleware.WhiteNoiseMiddleware"]
# Sentry
SENTRY_DSN = os.environ.get("SENTRY_DSN")
if SENTRY_DSN:
INSTALLED_APPS += ("raven.contrib.django.raven_compat",)
RAVEN_CONFIG = {
"dsn": SENTRY_DSN,
"release": os.environ.get("HEROKU_SLUG_COMMIT", ""),
}
ROOT_URLCONF = "config.urls"
ROOT_HOSTCONF = "config.hosts"
DEFAULT_HOST = "www"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [
os.path.join(BASE_DIR, "templates/"),
],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
"blog.context_processors.all",
],
},
},
]
WSGI_APPLICATION = "config.wsgi.application"
# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "simonwillisonblog",
},
"dashboard": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "simonwillisonblog",
"OPTIONS": {
"options": "-c default_transaction_read_only=on -c statement_timeout=3000"
},
},
}
# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_L10N = True
USE_TZ = True
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
DASHBOARD_ENABLE_FULL_EXPORT = True
if "DATABASE_URL" in os.environ:
# Parse database configuration from $DATABASE_URL
DATABASES["default"] = dj_database_url.config()
DATABASES["dashboard"] = dj_database_url.config()
DATABASES["dashboard"]["OPTIONS"] = {
"options": "-c default_transaction_read_only=on -c statement_timeout=3000"
}
if "DISABLE_AUTOCOMMIT" in os.environ:
DATABASES["default"]["AUTOCOMMIT"] = False
# Honor the 'X-Forwarded-Proto' header for request.is_secure()
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
# Allow all host headers
ALLOWED_HOSTS = ["*"]
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/
BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles")
STATIC_URL = "/static/"
STATICFILES_DIRS = (os.path.join(BASE_DIR, "static/"),)
# Simplified static file serving.
# https://warehouse.python.org/project/whitenoise/
STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
# urls.W002
# Your URL pattern '^/?archive/(\d{4})/(\d{2})/(\d{2})/$' has a regex beginning
# with a '/'. Remove this slash as it is unnecessary. If this pattern is
# targeted in an include(), ensure the include() pattern has a trailing '/'.
# This is deliberate (we get hits to //archive/ for some reason) so I'm
# silencing the warning:
SILENCED_SYSTEM_CHECKS = ("urls.W002",)
# Caching
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
}
}
djp.settings(globals())
</document_content>
</document>
<document index="6">
<source>./config/urls.py</source>
<document_content>
from django.urls import path, re_path, include
from django.contrib import admin
from django.http import (
HttpResponseRedirect,
HttpResponsePermanentRedirect,
HttpResponse,
)
from django.views.decorators.cache import never_cache
from django.conf import settings
import django_sql_dashboard
import djp
from blog import views as blog_views
from blog import search as search_views
from blog import tag_views
from blog import feeds
from feedstats.utils import count_subscribers
import os
import importlib.metadata
import json
from proxy.views import proxy_view
handler404 = "blog.views.custom_404"
def wellknown_webfinger(request):
remote_url = (
"https://fedi.simonwillison.net/.well-known/webfinger?"
+ request.META["QUERY_STRING"]
)
return proxy_view(request, remote_url)
def wellknown_hostmeta(request):
remote_url = (
"https://fedi.simonwillison.net/.well-known/host-meta?"
+ request.META["QUERY_STRING"]
)
return proxy_view(request, remote_url)
def wellknown_nodeinfo(request):
remote_url = "https://fedi.simonwillison.net/.well-known/nodeinfo"
return proxy_view(request, remote_url)
def username_redirect(request):
return HttpResponseRedirect("https://fedi.simonwillison.net/@simon")
def newsletter_redirect(request):
return HttpResponseRedirect("https://simonw.substack.com/")
def projects_redirect(request):
return HttpResponseRedirect(
"https://github.com/simonw/simonw/blob/main/releases.md"
)
FAVICON = open(os.path.join(settings.BASE_DIR, "static/favicon.ico"), "rb").read()
def static_redirect(request):
return HttpResponsePermanentRedirect(
"http://static.simonwillison.net%s" % request.get_full_path()
)
def tag_redirect(request, tag):
return HttpResponsePermanentRedirect("/tags/{}/".format(tag))
STAGING_ROBOTS_TXT = """
User-agent: Twitterbot
Disallow:
User-agent: *
Disallow: /
"""
PRODUCTION_ROBOTS_TXT = """
User-agent: ChatGPT-User
Disallow:
User-agent: *
Disallow: /admin/
Disallow: /search/
Sitemap: https://simonwillison.net/sitemap.xml
"""
def robots_txt(request):
if settings.STAGING:
txt = STAGING_ROBOTS_TXT
else:
txt = PRODUCTION_ROBOTS_TXT
return HttpResponse(txt, content_type="text/plain")
def favicon_ico(request):
return HttpResponse(FAVICON, content_type="image/x-icon")
@never_cache
def versions(request):
installed_packages = [
(dist.metadata["Name"], dist.version)
for dist in sorted(
importlib.metadata.distributions(), key=lambda d: d.metadata["Name"].lower()
)
]
return HttpResponse(
json.dumps(installed_packages, indent=4), content_type="text/plain"
)
urlpatterns = [
re_path(r"^$", blog_views.index),
re_path(r"^(\d{4})/$", blog_views.archive_year),
re_path(r"^(\d{4})/(\w{3})/$", blog_views.archive_month),
re_path(r"^(\d{4})/(\w{3})/(\d{1,2})/$", blog_views.archive_day),
re_path(r"^(\d{4})/(\w{3})/(\d{1,2})/([\-\w]+)/$", blog_views.archive_item),
re_path(r"^updates/(\d+)/$", blog_views.entry_updates),
re_path(r"^updates/(\d+)\.json$", blog_views.entry_updates_json),
# Redirects for entries, blogmarks, quotations by ID
re_path(r"^e/(\d+)/?$", blog_views.redirect_entry),
re_path(r"^b/(\d+)/?$", blog_views.redirect_blogmark),
re_path(r"^q/(\d+)/?$", blog_views.redirect_quotation),
# Ancient URL pattern still getting hits
re_path(r"^/?archive/(\d{4})/(\d{2})/(\d{2})/$", blog_views.archive_day_redirect),
re_path(
r"^/?archive/(\d{4})/(\d{2})/(\d{2})/([\-\w]+)/?$",
blog_views.archive_item_redirect,
),
# Fediverse
path(".well-known/webfinger", wellknown_webfinger),
path(".well-known/host-meta", wellknown_hostmeta),
path(".well-known/nodeinfo", wellknown_nodeinfo),
path("@simon", username_redirect),
re_path(r"^newsletter/?$", newsletter_redirect),
re_path(r"^projects/?$", projects_redirect),
re_path(r"^versions/$", versions),
re_path(r"^robots\.txt$", robots_txt),
re_path(r"^favicon\.ico$", favicon_ico),
re_path(r"^search/$", search_views.search),
re_path(r"^about/$", blog_views.about),
re_path(r"^tags/$", blog_views.tag_index),
re_path(r"^tags/(.*?)/$", blog_views.archive_tag),
re_path(r"^tags/(.*?).atom$", blog_views.archive_tag_atom),
re_path(r"^tag/([a-zA-Z0-9_-]+)/$", tag_redirect),
re_path(r"^series/$", blog_views.series_index),
re_path(r"^series/(.*?)/$", blog_views.archive_series),
re_path(r"^series/(.*?).atom$", blog_views.archive_series_atom),
re_path(r"^atom/entries/$", count_subscribers(feeds.Entries().__call__)),
re_path(r"^atom/links/$", count_subscribers(feeds.Blogmarks().__call__)),
re_path(r"^atom/everything/$", count_subscribers(feeds.Everything().__call__)),
re_path(r"^sitemap\.xml$", feeds.sitemap),
path("tools/", blog_views.tools),
path("tools/extract-title/", blog_views.tools_extract_title),
re_path(r"^tools/search-tags/$", search_views.tools_search_tags),
re_path(r"^write/$", blog_views.write),
# (r'^about/$', blog_views.about),
path("admin/bulk-tag/", blog_views.bulk_tag, name="bulk_tag"),
path("api/add-tag/", blog_views.api_add_tag, name="api_add_tag"),
re_path(r"^admin/", admin.site.urls),
re_path(r"^static/", static_redirect),
path("dashboard/", include(django_sql_dashboard.urls)),
path("user-from-cookies/", blog_views.user_from_cookies),
path("tags-autocomplete/", tag_views.tags_autocomplete),
] + djp.urlpatterns()
if settings.DEBUG:
try:
import debug_toolbar
urlpatterns = [
re_path(r"^__debug__/", include(debug_toolbar.urls))
] + urlpatterns
except ImportError:
pass
</document_content>
</document>
<document index="7">
<source>./config/urls_2003.py</source>
<document_content>
from django.urls import re_path, include
from django.http import (
HttpResponse,
)
from django.conf import settings
from blog import views_2003 as blog_views
DISALLOW_ALL = """
User-agent: *
Disallow: /
""".strip()
def robots_txt(request):
return HttpResponse(DISALLOW_ALL, content_type="text/plain")
urlpatterns = [
re_path(r"^$", blog_views.index),
re_path(r"^robots\.txt$", robots_txt),
]
if settings.DEBUG:
try:
import debug_toolbar
urlpatterns = [
re_path(r"^__debug__/", include(debug_toolbar.urls))
] + urlpatterns
except ImportError:
pass
</document_content>
</document>
<document index="8">
<source>./config/wsgi.py</source>
<document_content>
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
</document_content>
</document>
<document index="9">
<source>./projects/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="10">
<source>./projects/admin.py</source>
<document_content>
from django.contrib import admin
from .models import Project
admin.site.register(Project)
</document_content>
</document>
<document index="11">
<source>./projects/apps.py</source>
<document_content>
from django.apps import AppConfig
class ProjectsConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "projects"
</document_content>
</document>
<document index="12">
<source>./projects/models.py</source>
<document_content>
from django.db import models
class Project(models.Model):
# Constants for date precision
PRECISION_CHOICES = [
("D", "Day"),
("M", "Month"),
("Y", "Year"),
]
name = models.CharField(max_length=200)
image_url = models.URLField(max_length=200, null=True, blank=True)
url = models.URLField(max_length=200, null=True, blank=True)
short_description = models.TextField()
full_description = models.TextField()
start_date = models.DateField()
start_date_precision = models.CharField(
max_length=1, choices=PRECISION_CHOICES, default="D"
)
end_date = models.DateField(null=True, blank=True)
end_date_precision = models.CharField(
max_length=1, choices=PRECISION_CHOICES, default="D"
)
def __str__(self):
return self.name
def _display_date(self, date, precision):
if precision == "D":
return date.strftime("%d %B %Y") # Day precision, e.g. 05 March 2022
elif precision == "M":
return date.strftime("%B %Y") # Month precision, e.g. March 2022
else: # precision == 'Y'
return date.strftime("%Y") # Year precision, e.g. 2022
def display_start_date(self):
return self._display_date(self.start_date, self.start_date_precision)
def display_end_date(self):
if self.end_date:
return self._display_date(self.end_date, self.end_date_precision)
return ""
def display_date_range(self):
start_date = self.display_start_date()
end_date = self.display_end_date()
if start_date and end_date:
return f"{start_date} – {end_date}"
elif start_date:
return f"{start_date} onwards"
</document_content>
</document>
<document index="13">
<source>./projects/tests.py</source>
<document_content>
# Create your tests here.
</document_content>
</document>
<document index="14">
<source>./projects/views.py</source>
<document_content>
from django.shortcuts import render
from .models import Project
def index(request):
return render(
request,
"projects/index.html",
{"projects": Project.objects.all().order_by("-start_date")},
)
</document_content>
</document>
<document index="15">
<source>./projects/migrations/0001_initial.py</source>
<document_content>
# Generated by Django 4.1.7 on 2023-06-23 17:28
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Project",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=200)),
("image_url", models.URLField(blank=True, null=True)),
("url", models.URLField(blank=True, null=True)),
("short_description", models.TextField()),
("full_description", models.TextField()),
("start_date", models.DateField()),
(
"start_date_precision",
models.CharField(
choices=[("D", "Day"), ("M", "Month"), ("Y", "Year")],
default="D",
max_length=1,
),
),
("end_date", models.DateField(blank=True, null=True)),
(
"end_date_precision",
models.CharField(
choices=[("D", "Day"), ("M", "Month"), ("Y", "Year")],
default="D",
max_length=1,
),
),
],
),
]
</document_content>
</document>
<document index="16">
<source>./projects/migrations/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="17">
<source>./blog/__init__.py</source>
<document_content>
default_app_config = "blog.apps.BlogConfig"
</document_content>
</document>
<document index="18">
<source>./blog/admin.py</source>
<document_content>
from django.contrib import admin
from django.contrib.postgres.search import SearchQuery, SearchRank
from django.db.models.functions import Length
from django.db.models import F
from django import forms
from xml.etree import ElementTree
from .models import (
Entry,
Tag,
Quotation,
Blogmark,
Comment,
Series,
PreviousTagName,
LiveUpdate,
)
class BaseAdmin(admin.ModelAdmin):
date_hierarchy = "created"
raw_id_fields = ("tags",)
list_display = ("__str__", "slug", "created", "tag_summary")
list_filter = ("created",)
autocomplete_fields = ("tags",)
readonly_fields = ("import_ref",)
exclude = ("search_document",)
def get_queryset(self, request):
return super().get_queryset(request).prefetch_related("tags")
def get_search_results(self, request, queryset, search_term):
if not search_term:
return super().get_search_results(request, queryset, search_term)
query = SearchQuery(search_term, search_type="websearch")
rank = SearchRank(F("search_document"), query)
queryset = (
queryset.annotate(rank=rank).filter(search_document=query).order_by("-rank")
)
return queryset, False
class MyEntryForm(forms.ModelForm):
def clean_body(self):
# Ensure this is valid XML
body = self.cleaned_data["body"]
try:
ElementTree.fromstring("<entry>%s</entry>" % body)
except Exception as e:
raise forms.ValidationError(str(e))
return body
@admin.register(Entry)
class EntryAdmin(BaseAdmin):
form = MyEntryForm
prepopulated_fields = {"slug": ("title",)}
search_fields = ("title", "body")
list_filter = ("created", "series")
@admin.register(LiveUpdate)
class LiveUpdateAdmin(admin.ModelAdmin):
raw_id_fields = ("entry",)
@admin.register(Quotation)
class QuotationAdmin(BaseAdmin):
search_fields = ("tags__tag", "quotation")
list_display = ("__str__", "source", "created", "tag_summary")
prepopulated_fields = {"slug": ("source",)}
@admin.register(Blogmark)
class BlogmarkAdmin(BaseAdmin):
search_fields = ("tags__tag", "commentary")
prepopulated_fields = {"slug": ("link_title",)}
@admin.register(Tag)
class TagAdmin(admin.ModelAdmin):
search_fields = ("tag",)
def get_search_results(self, request, queryset, search_term):
search_term = search_term.strip()
if search_term:
return (
queryset.filter(tag__istartswith=search_term)
.annotate(tag_length=Length("tag"))
.order_by("tag_length"),
False,
)
else:
return queryset.all(), False
def save_model(self, request, obj, form, change):
if change:
old_obj = Tag.objects.get(pk=obj.pk)
if old_obj.tag != obj.tag:
PreviousTagName.objects.create(tag=obj, previous_name=old_obj.tag)
super().save_model(request, obj, form, change)
admin.site.register(
Comment,
list_filter=("created", "visible_on_site", "spam_status", "content_type"),
search_fields=("body", "name", "url", "email", "openid"),
list_display=(
"name",
"admin_summary",
"on_link",
"created",
"ip_link",
"visible_on_site",
"spam_status_options",
),
list_display_links=("name", "admin_summary"),
date_hierarchy="created",
)
admin.site.register(
Series,
list_display=(
"title",
"slug",
),
)
admin.site.register(
PreviousTagName, raw_id_fields=("tag",), list_display=("previous_name", "tag")
)
</document_content>
</document>
<document index="19">
<source>./blog/apps.py</source>
<document_content>
from django.apps import AppConfig
from django.db.models import signals
class BlogConfig(AppConfig):
name = "blog"
def ready(self):
# import signal handlers
from blog import signals
</document_content>
</document>
<document index="20">
<source>./blog/context_processors.py</source>
<document_content>
from blog.models import Entry, Blogmark, Quotation
from django.conf import settings
from django.core.cache import cache
def all(request):
return {
"years_with_content": years_with_content(),
}
def years_with_content():
cache_key = "years-with-content-2"
years = cache.get(cache_key)
if not years:
years = list(
set(
list(Entry.objects.datetimes("created", "year"))
+ list(Blogmark.objects.datetimes("created", "year"))
+ list(Quotation.objects.datetimes("created", "year"))
)
)
years.sort()
cache.set(cache_key, years, 60 * 60)
return years
</document_content>
</document>
<document index="21">
<source>./blog/factories.py</source>
<document_content>
import factory
import factory.django
import factory.fuzzy
from datetime import timezone
class BaseFactory(factory.django.DjangoModelFactory):
slug = factory.Sequence(lambda n: "slug%d" % n)
created = factory.Faker("past_datetime", tzinfo=timezone.utc)
class EntryFactory(BaseFactory):
class Meta:
model = "blog.Entry"
title = factory.Faker("sentence")
class BlogmarkFactory(BaseFactory):
class Meta:
model = "blog.Blogmark"
link_url = factory.Faker("uri")
link_title = factory.Faker("sentence")
commentary = factory.Faker("sentence")
class QuotationFactory(BaseFactory):
class Meta:
model = "blog.Quotation"
</document_content>
</document>
<document index="22">
<source>./blog/feeds.py</source>
<document_content>
from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Atom1Feed
from django.http import HttpResponse
from blog.models import Entry, Blogmark, Quotation
class Base(Feed):
feed_type = Atom1Feed
link = "/"
author_name = "Simon Willison"
def __call__(self, request, *args, **kwargs):
response = super(Base, self).__call__(request, *args, **kwargs)
# Open CORS headers
response["Access-Control-Allow-Origin"] = "*"
response["Access-Control-Allow-Methods"] = "GET, OPTIONS"
response["Access-Control-Max-Age"] = "1000"
# Tell CloudFlare to cache my feeds for 2 minutes
response["Cache-Control"] = "s-maxage=%d" % (2 * 60)
return response
def item_link(self, item):
return (
"https://simonwillison.net"
+ item.get_absolute_url()
+ "#atom-%s" % self.ga_source
)
def item_categories(self, item):
return [t.tag for t in item.tags.all()]
def item_pubdate(self, item):
return item.created
def item_updateddate(self, item):
return item.created
def get_feed(self, obj, request):
feedgen = super().get_feed(obj, request)
feedgen.content_type = "application/xml; charset=utf-8"
return feedgen
class Entries(Base):
title = "Simon Willison's Weblog: Entries"
ga_source = "entries"
def items(self):
return (
Entry.objects.filter(is_draft=False)
.prefetch_related("tags")
.order_by("-created")[:15]
)
def item_title(self, item):
return item.title
def item_description(self, item):
return item.body
class Blogmarks(Base):
title = "Simon Willison's Weblog: Blogmarks"
description_template = "feeds/blogmark.html"
ga_source = "blogmarks"
def items(self):
return (
Blogmark.objects.filter(is_draft=False)
.prefetch_related("tags")
.order_by("-created")[:15]
)
def item_title(self, item):
return item.link_title
class Everything(Base):
title = "Simon Willison's Weblog"
description_template = "feeds/everything.html"
ga_source = "everything"
def items(self):
# Pretty dumb implementation: pull top 30 of entries/blogmarks/quotations
# then sort them together and return most recent 30 combined
last_30_entries = list(
Entry.objects.filter(is_draft=False)
.prefetch_related("tags")
.order_by("-created")[:30]
)
last_30_blogmarks = list(
Blogmark.objects.filter(is_draft=False)
.prefetch_related("tags")
.order_by("-created")[:30]
)
last_30_quotations = list(
Quotation.objects.filter(is_draft=False)
.prefetch_related("tags")
.order_by("-created")[:30]
)
combined = last_30_blogmarks + last_30_entries + last_30_quotations
combined.sort(key=lambda e: e.created, reverse=True)
return combined[:30]
def item_title(self, item):
if isinstance(item, Entry):
return item.title
elif isinstance(item, Blogmark):
return item.link_title
else:
return "Quoting %s" % item.source
class SeriesFeed(Everything):
ga_source = "series"
def __init__(self, series):
self.title = "Simon Willison's Weblog: {}".format(series.title)
self.series = series
def items(self):
return list(self.series.entry_set.all().order_by("-created"))
class EverythingTagged(Everything):
ga_source = "tag"
def __init__(self, title, items):
self.title = "Simon Willison's Weblog: {}".format(title)
self._items = items
def items(self):
return self._items
def sitemap(request):
xml = [
'<?xml version="1.0" encoding="UTF-8"?>'
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
]
for klass in (Entry, Blogmark, Quotation):
for obj in klass.objects.only("slug", "created"):
xml.append(
"<url><loc>https://simonwillison.net%s</loc></url>"
% obj.get_absolute_url()
)
xml.append("</urlset>")
return HttpResponse("\n".join(xml), content_type="application/xml")
</document_content>
</document>
<document index="23">
<source>./blog/middleware.py</source>
<document_content>
from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse
from django.shortcuts import redirect
class AmpersandRedirectMiddleware:
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
full_path = request.get_full_path()
if "&" in full_path or "&%3B" in full_path:
parsed_url = urlparse(full_path)
query_params = parse_qsl(parsed_url.query)
# Replace & with & in the query parameters
corrected_query = [(k.replace("amp;", ""), v) for k, v in query_params]
# Rebuild the URL with corrected query parameters
corrected_url = urlunparse(
(
parsed_url.scheme,
parsed_url.netloc,
parsed_url.path,
parsed_url.params,
urlencode(corrected_query),
parsed_url.fragment,
)
)
# Redirect the user to the corrected URL
return redirect(corrected_url)
response = self.get_response(request)
return response
</document_content>
</document>
<document index="24">
<source>./blog/models.py</source>
<document_content>
from django.db import models
from django.db.models import Sum, Subquery, OuterRef, IntegerField
from django.utils.dates import MONTHS_3
from django.db.models.functions import Coalesce
from django.utils.safestring import mark_safe
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.db.models import JSONField, Count
from django.contrib.postgres.search import SearchVectorField
from django.contrib.postgres.indexes import GinIndex
from django.utils.html import escape, strip_tags
from collections import Counter
import re
import arrow
import datetime
from markdown import markdown
from xml.etree import ElementTree
tag_re = re.compile("^[a-z0-9]+$")
class Tag(models.Model):
tag = models.SlugField(unique=True)
description = models.TextField(blank=True)
def __str__(self):
return self.tag
def description_rendered(self):
if self.description:
return mark_safe(markdown(self.description))
else:
return ""
def get_absolute_url(self):
return "/tags/%s/" % self.tag
def get_link(self, reltag=False):
return mark_safe(
'<a href="%s"%s>%s</a>'
% (self.get_absolute_url(), (reltag and ' rel="tag"' or ""), self)
)
def get_reltag(self):
return self.get_link(reltag=True)
def entry_count(self):
return self.entry_set.filter(is_draft=False).count()
def link_count(self):
return self.blogmark_set.filter(is_draft=False).count()
def quote_count(self):
return self.quotation_set.filter(is_draft=False).count()
def total_count(self):
entry_count = Subquery(
Entry.objects.filter(is_draft=False, tags=OuterRef("pk"))
.values("tags")
.annotate(count=Count("id"))
.values("count"),
output_field=IntegerField(),
)
blogmark_count = Subquery(
Blogmark.objects.filter(is_draft=False, tags=OuterRef("pk"))
.values("tags")
.annotate(count=Count("id"))
.values("count"),
output_field=IntegerField(),
)
quotation_count = Subquery(
Quotation.objects.filter(is_draft=False, tags=OuterRef("pk"))
.values("tags")
.annotate(count=Count("id"))
.values("count"),
output_field=IntegerField(),
)
result = (
Tag.objects.filter(pk=self.pk)
.annotate(
total_count=Sum(
Coalesce(entry_count, 0)
+ Coalesce(blogmark_count, 0)
+ Coalesce(quotation_count, 0)
)
)
.values("total_count")
.first()
)
return result["total_count"] if result else 0
def all_types_queryset(self):
entries = (
self.entry_set.all()
.annotate(type=models.Value("entry", output_field=models.CharField()))
.values("pk", "created", "type")
)
blogmarks = (
self.blogmark_set.all()
.annotate(type=models.Value("blogmark", output_field=models.CharField()))
.values("pk", "created", "type")
)
quotations = (
self.quotation_set.all()
.annotate(type=models.Value("quotation", output_field=models.CharField()))
.values("pk", "created", "type")
)
return entries.union(blogmarks, quotations).order_by("-created")
def get_related_tags(self, limit=10):
"""Get all items tagged with this, look at /their/ tags, order by count"""
if not hasattr(self, "_related_tags"):
counts = Counter()
for klass, collection in (
(Entry, "entry_set"),
(Blogmark, "blogmark_set"),
(Quotation, "quotation_set"),
):
qs = klass.objects.filter(
pk__in=getattr(self, collection).all()
).values_list("tags__tag", flat=True)
counts.update(t for t in qs if t != self.tag)
tag_names = [p[0] for p in counts.most_common(limit)]
tags_by_name = {t.tag: t for t in Tag.objects.filter(tag__in=tag_names)}
# Need a list in the correct order
self._related_tags = [tags_by_name[name] for name in tag_names]
return self._related_tags
def rename_tag(self, new_name):
PreviousTagName.objects.create(tag=self, previous_name=self.tag)
self.tag = new_name
self.save()
class PreviousTagName(models.Model):
tag = models.ForeignKey(Tag, on_delete=models.CASCADE)
previous_name = models.SlugField()
def __str__(self):
return self.previous_name
class Series(models.Model):
created = models.DateTimeField(default=datetime.datetime.utcnow)
slug = models.SlugField(max_length=64, unique=True)
title = models.CharField(max_length=255)
summary = models.TextField()
def summary_rendered(self):
if self.summary:
return mark_safe(markdown(self.summary))
else:
return ""
def entries_ordest_first(self):
return self.entry_set.order_by("created")
def get_absolute_url(self):
return "/series/{}/".format(self.slug)
def __str__(self):
return self.title
class Meta:
verbose_name_plural = "Series"
class BaseModel(models.Model):
created = models.DateTimeField(default=datetime.datetime.utcnow)
tags = models.ManyToManyField(Tag, blank=True)
slug = models.SlugField(max_length=64)
metadata = JSONField(blank=True, default=dict)
search_document = SearchVectorField(null=True)
import_ref = models.TextField(max_length=64, null=True, unique=True)
card_image = models.CharField(max_length=128, null=True, blank=True)
series = models.ForeignKey(Series, blank=True, null=True, on_delete=models.PROTECT)
is_draft = models.BooleanField(default=False) # P9163
def created_unixtimestamp(self):
return int(arrow.get(self.created).timestamp())
def tag_summary(self):
return " ".join(t.tag for t in self.tags.all())
def get_absolute_url(self):
return "/%d/%s/%d/%s/" % (
self.created.year,
MONTHS_3[self.created.month].title(),
self.created.day,
self.slug,
)
def edit_url(self):
return "/admin/blog/%s/%d/" % (self.__class__.__name__.lower(), self.id)
class Meta:
abstract = True
ordering = ("-created",)
indexes = [GinIndex(fields=["search_document"])]
class Entry(BaseModel):
title = models.CharField(max_length=255)
body = models.TextField()
tweet_html = models.TextField(
blank=True,
null=True,
help_text="""
Paste in the embed tweet HTML, minus the script tag,
to display a tweet in the sidebar next to this entry.
""".strip(),
)
extra_head_html = models.TextField(
blank=True,
null=True,
help_text="""
Extra HTML to be included in the <head> for this entry
""".strip(),
)
custom_template = models.CharField(max_length=100, null=True, blank=True)
is_entry = True
live_timezone = models.CharField(max_length=100, null=True, blank=True)
def next_by_created(self):
return super().get_next_by_created(is_draft=False)
def previous_by_created(self):
return super().get_previous_by_created(is_draft=False)
def images(self):
"""Extracts images from entry.body"""
et = ElementTree.fromstring("<entry>%s</entry>" % self.body)
return [i.attrib for i in et.findall(".//img")]
def index_components(self):
return {
"A": self.title,
"C": strip_tags(self.body),
"B": " ".join(self.tags.values_list("tag", flat=True)),
}
def series_info(self):
entries = list(self.series.entries_ordest_first().defer("body"))
has_next = False
start = 1
# If there are more than 7, only show 3 before and 3 after this one
if len(entries) > 7:
entry_ids = [e.pk for e in entries]
this_index = entry_ids.index(self.pk)
if this_index < 4:
entries = entries[:7]
start = 1
else:
entries = entries[this_index - 3 : this_index + 4]
start = (this_index - 3) + 1
has_next = len(entry_ids) > start + len(entries) - 1
return {
"start": start,
"entries": entries,
"has_next": has_next,
}
def multi_paragraph(self):
return self.body.count("<p") > 1
def __str__(self):
return self.title
class Meta(BaseModel.Meta):
verbose_name_plural = "Entries"
class LiveUpdate(models.Model):
created = models.DateTimeField(auto_now_add=True)
content = models.TextField()
entry = models.ForeignKey(Entry, related_name="updates", on_delete=models.CASCADE)
def __str__(self):
return "{}: {}".format(self.created, self.content)
class Quotation(BaseModel):
quotation = models.TextField()
source = models.CharField(max_length=255)
source_url = models.URLField(
blank=True,
null=True,
)
context = models.CharField(max_length=255, blank=True, null=True)
is_quotation = True
def body(self):
return mark_safe(markdown(self.quotation))
def body_strip_tags(self):
return strip_tags(markdown(self.quotation))
def context_rendered(self):
if self.context:
rendered = markdown(self.context)
# Remove leading/trailing <p> tag
if rendered.startswith("<p>") and rendered.endswith("</p>"):
return mark_safe(rendered[3:-4])
return mark_safe(rendered)
else:
return ""
def title(self):
"""Mainly a convenience for the comments RSS feed"""
return "A quote from %s" % escape(self.source)
def index_components(self):
return {
"A": self.quotation,
"B": " ".join(self.tags.values_list("tag", flat=True)),
"C": self.source,
}
def __str__(self):
return self.body_strip_tags()
class Blogmark(BaseModel):
link_url = models.URLField(max_length=512)
link_title = models.CharField(max_length=255)
via_url = models.URLField(blank=True, null=True, max_length=512)
via_title = models.CharField(max_length=255, blank=True, null=True)
commentary = models.TextField()
use_markdown = models.BooleanField(
default=False,
help_text='Images can use the img element - set width="..." for a specific width and use class="blogmark-image" to center and add a drop shadow.',
)
is_blogmark = True
def index_components(self):
return {
"A": self.link_title,
"B": " ".join(self.tags.values_list("tag", flat=True)),
"C": self.commentary
+ " "
+ self.link_domain()
+ " "
+ (self.via_title or ""),
}
def __str__(self):
return self.link_title
def link_domain(self):
return self.link_url.split("/")[2]
def body(self):
if self.use_markdown:
return mark_safe(markdown(self.commentary))
return self.commentary
def word_count(self):
count = len(self.commentary.split())
if count == 1:
return "1 word"
else:
return "%d words" % count
class Photo(models.Model):
flickr_id = models.CharField(max_length=32)
server = models.CharField(max_length=8)
secret = models.CharField(max_length=32)
title = models.CharField(max_length=255, blank=True, null=True)
longitude = models.CharField(max_length=32, blank=True, null=True)
latitude = models.CharField(max_length=32, blank=True, null=True)
created = models.DateTimeField()
def __str__(self):
return self.title
def photopage(self):
return "http://www.flickr.com/photo.gne?id=%s" % self.flickr_id
def url_s(self):
return "http://static.flickr.com/%s/%s_%s_s.jpg" % (
self.server,
self.flickr_id,
self.secret,
)
def view_thumb(self):
return '<a href="%s"><img src="%s" width="75" height="75" /></a>' % (
self.photopage(),
self.url_s(),
)
class Photoset(models.Model):
flickr_id = models.CharField(max_length=32)
title = models.CharField(max_length=255, blank=True, null=True)
description = models.TextField()
photos = models.ManyToManyField(
Photo,
related_name="in_photoset",
)
primary = models.ForeignKey(Photo, on_delete=models.CASCADE)
def __str__(self):
return self.title
def get_absolute_url(self):
return "http://www.flickr.com/photos/simon/sets/%s/" % self.flickr_id
def view_thumb(self):
return '<a href="%s"><img src="%s" width="75" height="75" /></a>' % (
self.primary.photopage(),
self.primary.url_s(),
)
def has_map(self):
return self.photos.filter(longitude__isnull=False).count() > 0
BAD_WORDS = (
"viagra",
"cialis",
"poker",
"levitra",
"casino",
"ifrance.com",
"phentermine",
"plasmatics.com",
"xenical",
"sohbet",
"oyuna",
"oyunlar",
)
SPAM_STATUS_OPTIONS = (
("normal", "Not suspected"),
("approved", "Approved"),
("suspected", "Suspected"),
("spam", "SPAM"),
)
COMMENTS_ALLOWED_ON = ("entry", "blogmark", "quotation")
class Comment(models.Model):
content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
object_id = models.PositiveIntegerField(db_index=True)
item = GenericForeignKey()
# The comment
body = models.TextField()
created = models.DateTimeField()
# Author information
name = models.CharField(max_length=50)
url = models.URLField(max_length=255, blank=True, null=True)
email = models.CharField(max_length=50, blank=True, null=True)
openid = models.CharField(max_length=255, blank=True, null=True)
ip = models.GenericIPAddressField()
# Spam filtering
spam_status = models.CharField(max_length=16, choices=SPAM_STATUS_OPTIONS)
visible_on_site = models.BooleanField(default=True, db_index=True)
spam_reason = models.TextField()
def get_absolute_url(self):
return "/%d/%s/%d/%s/#c%d" % (
self.item.created.year,
MONTHS_3[self.item.created.month].title(),
self.item.created.day,
self.item.slug,
self.id,
)
def edit_url(self):
return "/admin/blog/comment/%d/" % self.id
def __str__(self):
return '%s on "%s"' % (self.name, self.item)
def admin_summary(self):
return '<b>%s</b><br><span style="color: black;">%s</span>' % (
escape(str(self)),
escape(self.body[:200]),
)
admin_summary.allow_tags = True
admin_summary.short_description = "Comment"
def on_link(self):
return '<a href="%s">%s</a>(<a href="%s">#</a>)' % (
self.item.get_absolute_url(),
self.content_type.name.title(),
self.get_absolute_url(),
)
on_link.allow_tags = True
on_link.short_description = "On"
def ip_link(self):
return '<a href="/admin/blog/comment/?ip__exact=%s">%s</a>' % (self.ip, self.ip)
ip_link.allow_tags = True
ip_link.short_description = "IP"
def spam_status_options(self):
bits = []
bits.append(self.get_spam_status_display())
bits.append("<br>")
bits.append(
'<form class="flagspam" action="/admin/flagspam/" ' + ' method="post">'
)
bits.append('<input type="hidden" name="id" value="%s">' % self.id)
bits.append(
'<input type="submit" class="submit" '
+ 'name="flag_as_spam" value="SPAM"> '
)
bits.append(
'<input type="submit" class="submit" '
+ 'name="flag_as_approved" value="OK">'
)
bits.append("</form>")
return "".join(bits)
spam_status_options.allow_tags = True
spam_status_options.short_description = "Spam status"
class Meta:
ordering = ("-created",)
get_latest_by = "created"
def load_mixed_objects(dicts):
"""
Takes a list of dictionaries, each of which must at least have a 'type'
and a 'pk' key. Returns a list of ORM objects of those various types.
Each returned ORM object has a .original_dict attribute populated.
"""
to_fetch = {}
for d in dicts:
to_fetch.setdefault(d["type"], set()).add(d["pk"])
fetched = {}
for key, model in (
("blogmark", Blogmark),
("entry", Entry),
("quotation", Quotation),
):
ids = to_fetch.get(key) or []
objects = model.objects.prefetch_related("tags").filter(pk__in=ids)
for obj in objects:
fetched[(key, obj.pk)] = obj
# Build list in same order as dicts argument
to_return = []
for d in dicts:
item = fetched.get((d["type"], d["pk"])) or None
if item:
item.original_dict = d
to_return.append(item)
return to_return
</document_content>
</document>
<document index="25">
<source>./blog/search.py</source>
<document_content>
import time
import json
import re
from django.db import models
from django.db.models.functions import TruncYear, TruncMonth
from django.contrib.postgres.search import SearchQuery, SearchRank
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import HttpResponse, Http404
from django.shortcuts import render
from blog.models import Entry, Blogmark, Quotation, Tag, load_mixed_objects
from .views import MONTHS_3_REV_REV
from spellchecker import SpellChecker
import datetime
_spell = None
def get_spellchecker():
global _spell
if _spell is None:
_spell = SpellChecker()
# Load all tags into the spellchecker
tag_words = set()
for tag in Tag.objects.values_list("tag", flat=True):
tag_words.update(tag.split("-"))
_spell.word_frequency.load_words(tag_words)
return _spell
def get_suggestion(phrase):
words = phrase.split()
spell = get_spellchecker()
unknown = spell.unknown(words)
if not unknown:
return phrase
new_words = []
for word in words:
if word in unknown:
suggestion = spell.correction(word)
if suggestion:
new_words.append(suggestion)
else:
new_words.append(word)
else:
new_words.append(word)
return " ".join(new_words)
def parse_date_clauses(query):
from_date = None
to_date = None
from_match = re.search(r"from:(\d{4}-\d{2}-\d{2})", query)
to_match = re.search(r"to:(\d{4}-\d{2}-\d{2})", query)
if from_match:
from_date = datetime.datetime.strptime(from_match.group(1), "%Y-%m-%d").date()
if to_match:
to_date = datetime.datetime.strptime(to_match.group(1), "%Y-%m-%d").date()
query = re.sub(r"from:\d{4}-\d{2}-\d{2}", "", query)
query = re.sub(r"to:\d{4}-\d{2}-\d{2}", "", query)
return query.strip(), from_date, to_date
def search(request, q=None, return_context=False):
q = (q or request.GET.get("q", "")).strip()
search_q, from_date, to_date = parse_date_clauses(q)
search_q = search_q.strip()
start = time.time()
query = None
rank_annotation = None
if search_q:
query = SearchQuery(search_q, search_type="websearch")
rank_annotation = SearchRank(models.F("search_document"), query)
selected_tags = request.GET.getlist("tag")
if len(selected_tags) > 2:
return HttpResponse("Too many tags", status=400)
excluded_tags = request.GET.getlist("exclude.tag")
selected_type = request.GET.get("type", "")
selected_year = request.GET.get("year", "")
selected_month = request.GET.get("month", "")
values = ["pk", "type", "created"]
if search_q:
values.append("rank")
def make_queryset(klass, type_name):
qs = klass.objects.filter(is_draft=False).annotate(
type=models.Value(type_name, output_field=models.CharField())
)
if selected_year and selected_year.isdigit() and 2000 <= int(selected_year):
qs = qs.filter(created__year=int(selected_year))
if (
selected_month
and selected_month.isdigit()
and 1 <= int(selected_month) <= 12
):
qs = qs.filter(created__month=int(selected_month))
if from_date:
qs = qs.filter(created__gte=from_date)
if to_date:
qs = qs.filter(created__lt=to_date)
if search_q:
qs = qs.filter(search_document=query)
qs = qs.annotate(rank=rank_annotation)
for tag in selected_tags:
qs = qs.filter(tags__tag=tag)
for exclude_tag in excluded_tags:
qs = qs.exclude(tags__tag=exclude_tag)
return qs.order_by()
# Start with a .none() queryset just so we can union stuff onto it
qs = Entry.objects.annotate(
type=models.Value("empty", output_field=models.CharField())
)
if search_q:
qs = qs.annotate(rank=rank_annotation)
qs = qs.values(*values).none()
type_counts_raw = {}
tag_counts_raw = {}
year_counts_raw = {}
month_counts_raw = {}
for klass, type_name in (
(Entry, "entry"),
(Blogmark, "blogmark"),
(Quotation, "quotation"),
):
if selected_type and selected_type != type_name:
continue
klass_qs = make_queryset(klass, type_name)
type_count = klass_qs.count()
if type_count:
type_counts_raw[type_name] = type_count
for tag, count in (
Tag.objects.filter(**{"%s__in" % type_name: klass_qs})
.annotate(n=models.Count("tag"))
.values_list("tag", "n")
):
tag_counts_raw[tag] = tag_counts_raw.get(tag, 0) + count
for row in (
klass_qs.order_by()
.annotate(year=TruncYear("created"))
.values("year")
.annotate(n=models.Count("pk"))
):
year_counts_raw[row["year"]] = (
year_counts_raw.get(row["year"], 0) + row["n"]
)
# Only do month counts if a year is selected
if selected_year:
for row in (
klass_qs.order_by()
.annotate(month=TruncMonth("created"))
.values("month")
.annotate(n=models.Count("pk"))
):
month_counts_raw[row["month"]] = (
month_counts_raw.get(row["month"], 0) + row["n"]
)
qs = qs.union(klass_qs.values(*values))
sort = request.GET.get("sort")
if sort not in ("relevance", "date"):
sort = None
if sort is None:
if search_q:
sort = "relevance"
else:
sort = "date"
# can't sort by relevance if there's no search_q
if sort == "relevance" and not search_q:
sort = "date"
db_sort = {"relevance": "-rank", "date": "-created"}[sort]
qs = qs.order_by(db_sort)
type_counts = sorted(
[
{"type": type_name, "n": value}
for type_name, value in list(type_counts_raw.items())
],
key=lambda t: t["n"],
reverse=True,
)
tag_counts = sorted(
[{"tag": tag, "n": value} for tag, value in list(tag_counts_raw.items())],
key=lambda t: t["n"],
reverse=True,
)[:40]
year_counts = sorted(
[{"year": year, "n": value} for year, value in list(year_counts_raw.items())],
key=lambda t: t["year"],
)
month_counts = sorted(
[
{"month": month, "n": value}
for month, value in list(month_counts_raw.items())
],
key=lambda t: t["month"],
)
paginator = Paginator(qs, 30)
page_number = request.GET.get("page") or "1"
try:
page = paginator.page(page_number)
except PageNotAnInteger:
raise Http404
except EmptyPage:
raise Http404
results = []
for obj in load_mixed_objects(page.object_list):
results.append(
{
"type": obj.original_dict["type"],
"rank": obj.original_dict.get("rank"),
"obj": obj,
}
)
end = time.time()
selected = {
"tags": selected_tags,
"year": selected_year,
"month": selected_month,
"type": selected_type,
"month_name": MONTHS_3_REV_REV.get(
selected_month and int(selected_month) or "", ""
).title(),
"from_date": from_date,
"to_date": to_date,
}
# Remove empty keys
selected = {key: value for key, value in list(selected.items()) if value}
# Dynamic title
noun = {
"quotation": "Quotations",
"blogmark": "Blogmarks",
"entry": "Entries",
}.get(selected.get("type")) or "Posts"
title = noun
if search_q:
title = "“%s” in %s" % (search_q, title.lower())
if selected.get("tags"):
title += " tagged %s" % (", ".join(selected["tags"]))
datebits = []
if selected.get("month_name"):
datebits.append(selected["month_name"])
if selected.get("year"):
datebits.append(selected["year"])
if datebits:
title += " in %s" % (", ".join(datebits))
if from_date or to_date:
date_range = []
if from_date:
date_range.append(f"from {from_date}")
if to_date:
date_range.append(f"to {to_date}")
title += " " + " ".join(date_range)
if not search_q and not selected:
title = "Search"
# if no results, count how many a spell-corrected search would get
suggestion = None
num_corrected_results = 0
if not results and search_q and not return_context:
suggestion = get_suggestion(search_q)
corrected_context = search(request, suggestion, return_context=True)
num_corrected_results = corrected_context["total"]
context = {
"q": q,
"sort": sort,
"title": title,
"results": results,
"total": paginator.count,
"page": page,
"duration": end - start,
"type_counts": type_counts,
"tag_counts": tag_counts,
"year_counts": year_counts,
"month_counts": month_counts,
"selected_tags": selected_tags,
"excluded_tags": excluded_tags,
"selected": selected,
"suggestion": suggestion,
"num_corrected_results": num_corrected_results,
}
if return_context:
return context
else:
return render(request, "search.html", context)
def tools_search_tags(request):
q = request.GET.get("q", "").strip()
results = []
if q:
results = list(
Tag.objects.filter(tag__icontains=q).values_list("tag", flat=True)
)
results.sort(key=lambda t: len(t))
return HttpResponse(json.dumps({"tags": results}), content_type="application/json")
</document_content>
</document>
<document index="26">
<source>./blog/signals.py</source>
<document_content>
from django.dispatch import receiver
from django.db.models.signals import post_save, m2m_changed
from django.db.models import Value, TextField
from django.contrib.postgres.search import SearchVector
from django.db import transaction
from blog.models import BaseModel, Tag
import operator
from functools import reduce
@receiver(post_save)
def on_save(sender, **kwargs):
if not issubclass(sender, BaseModel):
return
transaction.on_commit(make_updater(kwargs["instance"]))
@receiver(m2m_changed)
def on_m2m_changed(sender, **kwargs):
instance = kwargs["instance"]
model = kwargs["model"]
if model is Tag:
transaction.on_commit(make_updater(instance))
elif isinstance(instance, Tag):
for obj in model.objects.filter(pk__in=kwargs["pk_set"]):
transaction.on_commit(make_updater(obj))
def make_updater(instance):
components = instance.index_components()
pk = instance.pk
def on_commit():
search_vectors = []
for weight, text in list(components.items()):
search_vectors.append(
SearchVector(Value(text, output_field=TextField()), weight=weight)
)
instance.__class__.objects.filter(pk=pk).update(
search_document=reduce(operator.add, search_vectors)
)
return on_commit
</document_content>
</document>
<document index="27">
<source>./blog/tag_views.py</source>
<document_content>
from .models import Tag
from django.db.models import (
Case,
When,
Value,
IntegerField,
F,
Q,
Subquery,
OuterRef,
Count,
)
from django.db.models.functions import Length
from django.http import JsonResponse, HttpResponse
import json
def tags_autocomplete(request):
query = request.GET.get("q", "")
# Remove whitespace
query = "".join(query.split())
if query:
entry_count = (
Tag.objects.filter(id=OuterRef("pk"))
.annotate(
count=Count("entry", filter=Q(entry__is_draft=False), distinct=True)
)
.values("count")
)
# Subquery for counting blogmarks
blogmark_count = (
Tag.objects.filter(id=OuterRef("pk"))
.annotate(
count=Count(
"blogmark", filter=Q(blogmark__is_draft=False), distinct=True
)
)
.values("count")
)
# Subquery for counting quotations
quotation_count = (
Tag.objects.filter(id=OuterRef("pk"))
.annotate(
count=Count(
"quotation", filter=Q(quotation__is_draft=False), distinct=True
)
)
.values("count")
)
tags = (
Tag.objects.filter(tag__icontains=query)
.annotate(
total_entry=Subquery(entry_count),
total_blogmark=Subquery(blogmark_count),
total_quotation=Subquery(quotation_count),
is_exact_match=Case(
When(tag__iexact=query, then=Value(1)),
default=Value(0),
output_field=IntegerField(),
),
)
.annotate(
count=F("total_entry") + F("total_blogmark") + F("total_quotation")
)
.order_by("-is_exact_match", "-count", Length("tag"))[:5]
)
else:
tags = Tag.objects.none()
if request.GET.get("debug"):
return HttpResponse(
"<html><body><pre>"
+ json.dumps(list(tags.values()), indent=4)
+ "</pre><hr><code>"
+ str(tags.query)
+ "</body></html>"
)
return JsonResponse({"tags": list(tags.values())})
</document_content>
</document>
<document index="28">
<source>./blog/tests.py</source>
<document_content>
from django.test import TransactionTestCase
from blog.templatetags.entry_tags import do_typography_string
from .factories import (
EntryFactory,
BlogmarkFactory,
QuotationFactory,
)
from blog.models import Tag, PreviousTagName
import json
class BlogTests(TransactionTestCase):
def test_homepage(self):
db_entries = [
EntryFactory(),
EntryFactory(),
EntryFactory(),
]
BlogmarkFactory()
QuotationFactory()
response = self.client.get("/")
entries = response.context["entries"]
self.assertEqual(
[e.pk for e in entries],
[e.pk for e in sorted(db_entries, key=lambda e: e.created, reverse=True)],
)
def test_django_header_plugin(self):
response = self.client.get("/")
self.assertIn("Django-Composition", response)
def test_other_pages(self):
entry = EntryFactory()
blogmark = BlogmarkFactory()
quotation = QuotationFactory()
for path in (
"/",
"/{}/".format(entry.created.year),
entry.get_absolute_url(),
blogmark.get_absolute_url(),
quotation.get_absolute_url(),
"/{}/".format(entry.created.year),
"/atom/everything/",
):
response = self.client.get(path)
assert response.status_code == 200
def test_entry(self):
entry = EntryFactory()
response = self.client.get(entry.get_absolute_url())
self.assertTemplateUsed(response, "entry.html")
self.assertEqual(response.context["entry"].pk, entry.pk)
def test_blogmark(self):
blogmark = BlogmarkFactory()
response = self.client.get(blogmark.get_absolute_url())
self.assertTemplateUsed(response, "blogmark.html")
self.assertEqual(response.context["blogmark"].pk, blogmark.pk)
def test_quotation(self):
quotation = QuotationFactory()
response = self.client.get(quotation.get_absolute_url())
self.assertTemplateUsed(response, "quotation.html")
self.assertEqual(response.context["quotation"].pk, quotation.pk)
def test_archive_year(self):
quotation = QuotationFactory()
response = self.client.get("/{}/".format(quotation.created.year))
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "archive_year.html")
def test_markup(self):
entry = EntryFactory(
title="Hello & goodbye",
body="<p>First paragraph</p><p>Second paragraph</p>",
)
response = self.client.get(entry.get_absolute_url())
self.assertContains(
response,
"""
<h2>Hello & goodbye</h2>
""",
html=True,
)
self.assertContains(
response,
"""
<p>First paragraph</p><p>Second paragraph</p>
""".strip(),
)
def test_update_blogmark_runs_commit_hooks(self):
# This was throwing errors on upgrade Django 2.2 to 2.2.1
blogmark = BlogmarkFactory()
assert blogmark.pk
blogmark.commentary = "hello there"
blogmark.save()
def test_do_typography_string(self):
for input, expected in (
("Hello, world", "Hello, world"),
('Hello, "world"!', "Hello, “world”!"),
("Hello, world's!", "Hello, world’s!"),
('Hello, <"world"!', "Hello, <“world”!"),
# Do not do these ones:
('Hello, <"world">!', 'Hello, <"world">!'),
("Hello, <'world'>!", "Hello, <'world'>!"),
# This caused a recursion error at one point
(
"""Should you pin your library's dependencies using "click>=7,<8" or "click~=7.0"? Henry Schreiner's short answer is no, and his long answer is an exhaustive essay covering every conceivable aspect of this thorny Python packaging problem.""",
'Should you pin your library\'s dependencies using "click>=7,<8" or "click~=7.0"? Henry Schreiner\'s short answer is no, and his long answer is an exhaustive essay covering every conceivable aspect of this thorny Python packaging problem.',
),
):
self.assertEqual(do_typography_string(input), expected)
def test_rename_tag_creates_previous_tag_name(self):
tag = Tag.objects.create(tag="old-name")
tag.entry_set.create(
title="Test entry",
body="Test entry body",
created="2020-01-01T00:00:00+00:00",
)
assert self.client.get("/tags/old-name/").status_code == 200
assert self.client.get("/tags/new-name/").status_code == 404
tag.rename_tag("new-name")
self.assertEqual(tag.tag, "new-name")
previous_tag_name = PreviousTagName.objects.get(tag=tag)
self.assertEqual(previous_tag_name.previous_name, "old-name")
assert self.client.get("/tags/old-name/").status_code == 301
assert self.client.get("/tags/new-name/").status_code == 200
def test_tag_with_hyphen(self):
tag = Tag.objects.create(tag="tag-with-hyphen")
self.assertEqual(tag.tag, "tag-with-hyphen")
def test_draft_items_not_displayed(self):
draft_entry = EntryFactory(is_draft=True, title="draftentry")
draft_blogmark = BlogmarkFactory(is_draft=True, link_title="draftblogmark")
draft_quotation = QuotationFactory(is_draft=True, source="draftquotation")
testing = Tag.objects.get_or_create(tag="testing")[0]
live_entry = EntryFactory(title="publishedentry", created=draft_entry.created)
live_blogmark = BlogmarkFactory(
link_title="publishedblogmark", created=draft_blogmark.created
)
live_quotation = QuotationFactory(
source="publishedquotation", created=draft_quotation.created
)
for obj in (
draft_entry,
draft_blogmark,
draft_quotation,
live_entry,
live_blogmark,
live_quotation,
):
obj.tags.add(testing)
paths = (
"/", # Homepage
"/{}/".format(draft_entry.created.year),
"/{}/{}/".format(
draft_entry.created.year, draft_entry.created.strftime("%b")
),
"/{}/{}/{}/".format(
draft_entry.created.year,
draft_entry.created.strftime("%b"),
draft_entry.created.day,
),
"/search/?q=testing",
"/tags/testing/",
live_entry.get_absolute_url(),
)
counts = json.loads(self.client.get("/tags-autocomplete/?q=testing").content)
assert counts == {
"tags": [
{
"id": testing.pk,
"tag": "testing",
"description": "",
"total_entry": 1,
"total_blogmark": 1,
"total_quotation": 1,
"is_exact_match": 1,
"count": 3,
}
]
}
for path in paths:
response = self.client.get(path)
self.assertNotContains(response, "draftentry")
robots_fragment = '<meta name="robots" content="noindex">'
draft_warning_fragment = "This is a draft post"
for obj in (draft_entry, draft_blogmark, draft_quotation):
response2 = self.client.get(obj.get_absolute_url())
self.assertContains(response2, robots_fragment)
self.assertContains(response2, draft_warning_fragment)
assert (
response2.headers["cache-control"]
== "private, no-cache, no-store, must-revalidate"
)
# Publish it
obj.is_draft = False
obj.save()
response3 = self.client.get(obj.get_absolute_url())
self.assertNotContains(response3, robots_fragment)
self.assertNotContains(response3, draft_warning_fragment)
assert "cache-control" not in response3.headers
counts2 = json.loads(self.client.get("/tags-autocomplete/?q=testing").content)
assert counts2 == {
"tags": [
{
"id": testing.pk,
"tag": "testing",
"description": "",
"total_entry": 2,
"total_blogmark": 2,
"total_quotation": 2,
"is_exact_match": 1,
"count": 6,
}
]
}
for path in paths:
response4 = self.client.get(path)
self.assertContains(response4, "draftentry")
def test_draft_items_not_in_feeds(self):
draft_entry = EntryFactory(is_draft=True, title="draftentry")
draft_blogmark = BlogmarkFactory(is_draft=True, link_title="draftblogmark")
draft_quotation = QuotationFactory(is_draft=True, source="draftquotation")
response1 = self.client.get("/atom/entries/")
self.assertNotContains(response1, draft_entry.title)
response2 = self.client.get("/atom/links/")
self.assertNotContains(response2, draft_blogmark.link_title)
response3 = self.client.get("/atom/everything/")
self.assertNotContains(response3, draft_entry.title)
self.assertNotContains(response3, draft_blogmark.link_title)
self.assertNotContains(response3, draft_quotation.source)
# Change draft status and check they show up
draft_entry.is_draft = False
draft_entry.save()
draft_blogmark.is_draft = False
draft_blogmark.save()
draft_quotation.is_draft = False
draft_quotation.save()
response4 = self.client.get("/atom/entries/")
self.assertContains(response4, draft_entry.title)
response5 = self.client.get("/atom/links/")
self.assertContains(response5, draft_blogmark.link_title)
response6 = self.client.get("/atom/everything/")
self.assertContains(response6, draft_entry.title)
self.assertContains(response6, draft_blogmark.link_title)
self.assertContains(response6, draft_quotation.source)
</document_content>
</document>
<document index="29">
<source>./blog/views.py</source>
<document_content>
# coding=utf8
from django.shortcuts import render, get_object_or_404
from django.http import JsonResponse
from django.contrib.admin.views.decorators import staff_member_required
from django.views.decorators.http import require_POST
from django.views.decorators.cache import never_cache
from django.db import models
from django.db.models import CharField, Value
from django.conf import settings
from django.core.paginator import (
Paginator,
EmptyPage,
PageNotAnInteger,
)
from django.http import Http404, HttpResponsePermanentRedirect as Redirect
from .models import (
Blogmark,
Entry,
Quotation,
Photo,
Photoset,
Series,
Tag,
PreviousTagName,
)
import requests
from bs4 import BeautifulSoup as Soup
import datetime
import random
from collections import Counter
import cloudflare
import os
import pytz
MONTHS_3_REV = {
"jan": 1,
"feb": 2,
"mar": 3,
"apr": 4,
"may": 5,
"jun": 6,
"jul": 7,
"aug": 8,
"sep": 9,
"oct": 10,
"nov": 11,
"dec": 12,
}
MONTHS_3_REV_REV = {value: key for key, value in list(MONTHS_3_REV.items())}
BLACKLISTED_TAGS = ("quora", "flash", "resolved", "recovered")
def set_no_cache(response):
response["Cache-Control"] = "private, no-cache, no-store, must-revalidate"
response["Pragma"] = "no-cache"
response["Expires"] = "0"
return response
def archive_item(request, year, month, day, slug):
if day.startswith("0"):
day = day.lstrip("0")
return Redirect("/%s/%s/%s/%s/" % (year, month, day, slug))
# This could be a quote OR link OR entry
for content_type, model in (
("blogmark", Blogmark),
("entry", Entry),
("quotation", Quotation),
):
try:
obj = get_object_or_404(
model,
created__year=int(year),
created__month=MONTHS_3_REV[month.lower()],
created__day=int(day),
slug=slug,
)
except Http404:
continue
# If item is entry posted before Dec 1 2006, add "previously hosted"
if content_type == "entry" and obj.created < datetime.datetime(
2006, 12, 1, 1, 1, 1, tzinfo=datetime.timezone.utc
):
previously_hosted = (
"http://simon.incutio.com/archive/"
+ obj.created.strftime("%Y/%m/%d/")
+ obj.slug
)
else:
previously_hosted = None
template = getattr(obj, "custom_template", None) or "{}.html".format(
content_type
)
updates = []
if isinstance(obj, Entry):
updates = list(obj.updates.order_by("created"))
for update in updates:
update.created_str = (
str(
update.created.astimezone(
pytz.timezone("America/Los_Angeles")
).time()
)
.split(".")[0]
.rsplit(":", 1)[0]
)
response = render(
request,
template,
{
content_type: obj,
"content_type": content_type,
"object_id": obj.id,
"previously_hosted": previously_hosted,
"item": obj,
"recent_articles": Entry.objects.filter(is_draft=False)
.prefetch_related("tags")
.order_by("-created")[0:3],
"is_draft": obj.is_draft,
"updates": updates,
},
)
if obj.is_draft:
set_no_cache(response)
return response
# If we get here, non of the views matched
raise Http404
def index(request):
# Get back 30 most recent across all item types
recent = list(
Entry.objects.filter(is_draft=False)
.annotate(content_type=Value("entry", output_field=CharField()))
.values("content_type", "id", "created")
.order_by()
.union(
Blogmark.objects.filter(is_draft=False)
.annotate(content_type=Value("blogmark", output_field=CharField()))
.values("content_type", "id", "created")
.order_by()
)
.union(
Quotation.objects.filter(is_draft=False)
.annotate(content_type=Value("quotation", output_field=CharField()))
.values("content_type", "id", "created")
.order_by()
)
.order_by("-created")[:30]
)
# Now load the entries, blogmarks, quotations
items = []
to_load = {}
for item in recent:
to_load.setdefault(item["content_type"], []).append(item["id"])
for content_type, model in (
("entry", Entry),
("blogmark", Blogmark),
("quotation", Quotation),
):
if content_type not in to_load:
continue
objects = model.objects.prefetch_related("tags").in_bulk(to_load[content_type])
items.extend([{"type": content_type, "obj": obj} for obj in objects.values()])
items.sort(key=lambda x: x["obj"].created, reverse=True)
response = render(
request,
"homepage.html",
{
"items": items,
"entries": Entry.objects.filter(is_draft=False)
.only("id", "slug", "created", "title", "extra_head_html")
.prefetch_related("tags")[0:40],
"current_tags": find_current_tags(5),
},
)
response["Cache-Control"] = "s-maxage=200"
return response
def entry_updates(request, entry_id):
entry = get_object_or_404(Entry, pk=entry_id)
updates = list(entry.updates.order_by("created"))
for update in updates:
update.created_str = (
str(update.created.astimezone(pytz.timezone("America/Los_Angeles")).time())
.split(".")[0]
.rsplit(":", 1)[0]
)
response = render(request, "entry_updates.html", {"updates": updates})
response["Cache-Control"] = "s-maxage=10"
return response
def entry_updates_json(request, entry_id):
entry = get_object_or_404(Entry, pk=entry_id)
updates = entry.updates.order_by("created")
since_id = request.GET.get("since")
if since_id:
updates = updates.filter(id__gt=since_id)
response = JsonResponse(
{
"updates": [
{
"id": update.id,
"created": update.created.isoformat(),
"created_str": (
str(
update.created.astimezone(
pytz.timezone("America/Los_Angeles")
).time()
)
.split(".")[0]
.rsplit(":", 1)[0]
),
"content": update.content,
}
for update in updates
]
}
)
response["Cache-Control"] = "s-maxage=10"
return response
def find_current_tags(num=5):
"""Returns num random tags from top 30 in recent 400 taggings"""
last_400_tags = list(
Tag.quotation_set.through.objects.annotate(
created=models.F("quotation__created")
)
.values("tag__tag", "created")
.union(
Tag.entry_set.through.objects.annotate(
created=models.F("entry__created")
).values("tag__tag", "created"),
Tag.blogmark_set.through.objects.annotate(
created=models.F("blogmark__created")
).values("tag__tag", "created"),
)
.order_by("-created")[:400]
)
counter = Counter(
t["tag__tag"] for t in last_400_tags if t["tag__tag"] not in BLACKLISTED_TAGS
)
candidates = [p[0] for p in counter.most_common(30)]
random.shuffle(candidates)
tags = Tag.objects.in_bulk(candidates[:num], field_name="tag")
return [tags[tag] for tag in candidates[:num]]
def archive_year(request, year):
year = int(year)
# Display list of months
# each with count of blogmarks/photos/entries/quotes
# We can cache this page heavily, so don't worry too much
months = []
max_count = 0
for month in range(1, 12 + 1):
date = datetime.date(year=year, month=month, day=1)
entry_count = Entry.objects.filter(
created__year=year, created__month=month, is_draft=False
).count()
link_count = Blogmark.objects.filter(
created__year=year, created__month=month, is_draft=False
).count()
quote_count = Quotation.objects.filter(
created__year=year, created__month=month, is_draft=False
).count()
photo_count = Photo.objects.filter(
created__year=year, created__month=month
).count()
month_count = entry_count + link_count + quote_count + photo_count
if month_count:
counts = [
("entry", entry_count),
("link", link_count),
("photo", photo_count),
("quote", quote_count),
]
counts_not_0 = [p for p in counts if p[1]]
months.append(
{
"date": date,
"counts": counts,
"counts_not_0": counts_not_0,
"entries": list(
Entry.objects.filter(
created__year=year, created__month=month, is_draft=False
).order_by("created")
),
}
)
max_count = max(
max_count, entry_count, link_count, quote_count, photo_count
)
return render(
request,
"archive_year.html",
{
"months": months,
"year": year,
"max_count": max_count,
},
)
def archive_month(request, year, month):
year = int(year)
month = MONTHS_3_REV[month.lower()]
items = []
from django.db import connection
for model, content_type in (
(Entry, "entry"),
(Quotation, "quotation"),
(Blogmark, "blogmark"),
):
ids = model.objects.filter(
created__year=year, created__month=month, is_draft=False
).values_list("id", flat=True)
items.extend(
[
{"type": content_type, "obj": obj}
for obj in list(
model.objects.prefetch_related("tags").in_bulk(ids).values()
)
]
)
if not items:
raise Http404
items.sort(key=lambda x: x["obj"].created)
# Paginate it
paginator = Paginator(items, min(1000, int(request.GET.get("size") or "30")))
page_number = request.GET.get("page") or "1"
if page_number == "last":
page_number = paginator.num_pages
try:
page = paginator.page(page_number)
except PageNotAnInteger:
raise Http404
except EmptyPage:
raise Http404
return render(
request,
"archive_month.html",
{
"items": page.object_list,
"total": paginator.count,
"page": page,
"date": datetime.date(year, month, 1),
},
)
def archive_day(request, year, month, day):
if day.startswith("0"):
day = day.lstrip("0")
return Redirect("/%s/%s/%s/" % (year, month, day))
context = {}
context["date"] = datetime.date(int(year), MONTHS_3_REV[month.lower()], int(day))
items = [] # Array of {'type': , 'obj': }
count = 0
for name, model in (
("blogmark", Blogmark),
("entry", Entry),
("quotation", Quotation),
):
filt = model.objects.filter(
created__year=int(year),
created__month=MONTHS_3_REV[month.lower()],
created__day=int(day),
is_draft=False,
).order_by("created")
context[name] = list(filt)
count += len(context[name])
items.extend([{"type": name, "obj": obj} for obj in context[name]])
# Now do photosets separately because they have no created field
context["photoset"] = list(
Photoset.objects.filter(
primary__created__year=int(year),
primary__created__month=MONTHS_3_REV[month.lower()],
primary__created__day=int(day),
)
)
for photoset in context["photoset"]:
photoset.created = photoset.primary.created
count += len(context["photoset"])
items.extend([{"type": "photoset", "obj": ps} for ps in context["photoset"]])
if count == 0:
raise Http404("No photosets/photos/entries/quotes/links for that day")
items.sort(key=lambda x: x["obj"].created)
context["items"] = items
photos = Photo.objects.filter(
created__year=context["date"].year,
created__month=context["date"].month,
created__day=context["date"].day,
)
context["photos"] = photos[:25]
# Should we show more_photos ?
if photos.count() > 25:
context["more_photos"] = photos.count()
return render(request, "archive_day.html", context)
def tag_index(request):
return render(request, "tags.html")
# This query gets the IDs of things that match all of the tags
INTERSECTION_SQL = """
SELECT %(content_table)s.id
FROM %(content_table)s, %(tag_table)s
WHERE is_draft = false AND %(tag_table)s.tag_id IN (
SELECT id FROM blog_tag WHERE tag IN (%(joined_tags)s)
)
AND %(tag_table)s.%(tag_table_content_key)s = %(content_table)s.id
GROUP BY %(content_table)s.id
HAVING COUNT(%(content_table)s.id) = %(tag_count)d
"""
def archive_tag(request, tags, atom=False):
from .feeds import EverythingTagged
tags_ = Tag.objects.filter(tag__in=tags.split("+")).values_list("tag", flat=True)[
:3
]
if not tags_:
# Try for a previous tag name
if "+" not in tags:
try:
previous = PreviousTagName.objects.get(previous_name=tags)
except PreviousTagName.DoesNotExist:
raise Http404
return Redirect("/tag/%s/" % previous.tag.tag)
raise Http404
tags = tags_
items = []
from django.db import connection
cursor = connection.cursor()
for model, content_type in (
(Entry, "entry"),
(Quotation, "quotation"),
(Blogmark, "blogmark"),
):
cursor.execute(
INTERSECTION_SQL
% {
"content_table": "blog_%s" % content_type,
"tag_table": "blog_%s_tags" % content_type,
"tag_table_content_key": "%s_id" % content_type,
"joined_tags": ", ".join(["'%s'" % tag for tag in tags]),
"tag_count": len(tags),
}
)
ids = [r[0] for r in cursor.fetchall()]
items.extend(
[
{"type": content_type, "obj": obj}
for obj in list(
model.objects.prefetch_related("tags").in_bulk(ids).values()
)
]
)
if not items:
raise Http404
items.sort(key=lambda x: x["obj"].created, reverse=True)
# Paginate it
paginator = Paginator(items, min(1000, int(request.GET.get("size") or "30")))
page_number = request.GET.get("page") or "1"
if page_number == "last":
page_number = paginator.num_pages
try:
page = paginator.page(page_number)
except PageNotAnInteger:
raise Http404
except EmptyPage:
raise Http404
if atom:
response = EverythingTagged(
", ".join(tags), (item["obj"] for item in page.object_list)
)(request)
# Pagination in link: header
if page.has_next():
query_dict = request.GET.copy()
query_dict["page"] = str(page.next_page_number())
next_url = request.path + "?" + query_dict.urlencode()
response["link"] = '<{}>; rel="next"'.format(next_url)
return response
return render(
request,
"archive_tag.html",
{
"tags": tags,
"items": page.object_list,
"total": paginator.count,
"page": page,
"only_one_tag": len(tags) == 1,
"tag": Tag.objects.get(tag=tags[0]),
},
)
def archive_tag_atom(request, tags):
return archive_tag(request, tags, atom=True)
def series_index(request):
return render(
request,
"series_index.html",
{
"all_series": Series.objects.all().annotate(
num_entries=models.Count("entry")
),
},
)
def archive_series(request, slug):
series = get_object_or_404(Series, slug=slug)
return render(
request,
"archive_series.html",
{
"series": series,
"items": [
{"type": "entry", "obj": obj}
for obj in series.entry_set.order_by("created").prefetch_related("tags")
],
},
)
def archive_series_atom(request, slug):
from .feeds import SeriesFeed
series = get_object_or_404(Series, slug=slug)
return SeriesFeed(series)(request)
@never_cache
@staff_member_required
def write(request):
return render(request, "write.html")
@never_cache
@staff_member_required
def tools(request):
if request.POST.get("purge_all"):
cf = cloudflare.CloudFlare(
email=settings.CLOUDFLARE_EMAIL, token=settings.CLOUDFLARE_TOKEN
)
cf.zones.purge_cache.delete(
settings.CLOUDFLARE_ZONE_ID, data={"purge_everything": True}
)
return Redirect(request.path + "?msg=Cache+purged")
return render(
request,
"tools.html",
{
"msg": request.GET.get("msg"),
"deployed_hash": os.environ.get("HEROKU_SLUG_COMMIT"),
},
)
@never_cache
@staff_member_required
def tools_extract_title(request):
url = request.GET.get("url", "")
if url:
soup = Soup(requests.get(url).content, "html5lib")
title = ""
title_el = soup.find("title")
if title_el:
title = title_el.text
return JsonResponse(
{
"title": title,
}
)
return JsonResponse({})
# Redirects for ancient patterns
# /archive/2002/10/24/
def archive_day_redirect(request, yyyy, mm, dd):
return Redirect("/%s/%s/%d/" % (yyyy, MONTHS_3_REV_REV[int(mm)].title(), int(dd)))
# /archive/2003/09/05/listamatic
def archive_item_redirect(request, yyyy, mm, dd, slug):
return Redirect(
"/%s/%s/%d/%s" % (yyyy, MONTHS_3_REV_REV[int(mm)].title(), int(dd), slug)
)
# For use with datasette-auth-existing-cookies
@never_cache
def user_from_cookies(request):
if not request.user.is_authenticated:
return JsonResponse({})
return JsonResponse(
{
"id": request.user.id,
"username": request.user.username,
"name": request.user.get_full_name(),
"is_staff": request.user.is_staff,
"is_superuser": request.user.is_superuser,
}
)
def redirect_entry(request, pk):
return Redirect(get_object_or_404(Entry, pk=pk).get_absolute_url())
def redirect_blogmark(request, pk):
return Redirect(get_object_or_404(Blogmark, pk=pk).get_absolute_url())
def redirect_quotation(request, pk):
return Redirect(get_object_or_404(Quotation, pk=pk).get_absolute_url())
def about(request):
return render(request, "about.html")
def custom_404(request, exception):
return render(
request,
"404.html",
{"q": [b.strip() for b in request.path.split("/") if b.strip()][-1]},
status=404,
)
@staff_member_required
@never_cache
def bulk_tag(request):
"""
Admin-only view for bulk tagging search results.
Reuses the search functionality but renders a custom template with tagging UI.
"""
from blog import search as search_views
context = search_views.search(request, return_context=True)
return render(request, "bulk_tag.html", context)
@require_POST
@staff_member_required
def api_add_tag(request):
"""
API endpoint to handle adding a tag to an object.
Expects content_type, object_id, and tag in the POST data.
"""
content_type = request.POST.get("content_type")
object_id = request.POST.get("object_id")
tag_name = request.POST.get("tag")
# Validate inputs
if not content_type or not object_id or not tag_name:
return JsonResponse({"error": "Missing required parameters"}, status=400)
# Get the object
model = {"entry": Entry, "blogmark": Blogmark, "quotation": Quotation}.get(
content_type
)
if not model:
return JsonResponse({"error": "Invalid content type"}, status=400)
try:
obj = model.objects.get(pk=object_id)
except model.DoesNotExist:
return JsonResponse({"error": "Object not found"}, status=404)
# Get or create the tag
tag = Tag.objects.get_or_create(tag=tag_name)[0]
# Add the tag to the object
obj.tags.add(tag)
return JsonResponse({"success": True, "tag": tag_name})
</document_content>
</document>
<document index="30">
<source>./blog/views_2003.py</source>
<document_content>
from django.shortcuts import render
from django.db.models import CharField, Value
import datetime
from .models import (
Blogmark,
Entry,
Quotation,
)
def index(request):
# Get back items across all item types - I went back to UK on 30 September 2004
cutoff_kwargs = {}
if request.GET.get("backdate"):
cutoff_kwargs["created__lte"] = datetime.datetime(2004, 9, 29, 0, 0, 0)
recent = list(
Entry.objects.filter(**cutoff_kwargs)
.annotate(content_type=Value("entry", output_field=CharField()))
.values("content_type", "id", "created")
.order_by()
.union(
Blogmark.objects.filter(**cutoff_kwargs)
.annotate(content_type=Value("blogmark", output_field=CharField()))
.values("content_type", "id", "created")
.order_by()
)
.union(
Quotation.objects.filter(**cutoff_kwargs)
.annotate(content_type=Value("quotation", output_field=CharField()))
.values("content_type", "id", "created")
.order_by()
)
.order_by("-created")[:30]
)
# Now load the entries, blogmarks, quotations
items = []
to_load = {}
for item in recent:
to_load.setdefault(item["content_type"], []).append(item["id"])
for content_type, model in (
("entry", Entry),
("blogmark", Blogmark),
("quotation", Quotation),
):
if content_type not in to_load:
continue
items.extend(
[
{"type": content_type, "obj": obj}
for obj in model.objects.in_bulk(to_load[content_type]).values()
]
)
items.sort(key=lambda x: x["obj"].created, reverse=True)
response = render(
request,
"homepage_2003.html",
{
"items": items,
},
)
response["Cache-Control"] = "s-maxage=200"
return response
</document_content>
</document>
<document index="31">
<source>./blog/templatetags/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="32">
<source>./blog/templatetags/blog_calendar.py</source>
<document_content>
from django import template
register = template.Library()
from blog.models import Entry, Photo, Quotation, Blogmark, Photoset
import datetime, copy
# This code used to use the following:
# import calendar
# calendar.Calendar().itermonthdates(date.year, date.month)
# But... that functionality of the calendar module is only available in
# 2.5, and relies on the with statement. D'oh!
FIRSTWEEKDAY = 0 # Monday
def itermonthdates(year, month):
"Modelled after 2.5's calendar...itermonthdates"
date = datetime.date(year, month, 1)
# Go back to the beginning of the week
days = (date.weekday() - FIRSTWEEKDAY) % 7
date -= datetime.timedelta(days=days)
oneday = datetime.timedelta(days=1)
while True:
yield date
date += oneday
if date.month != month and date.weekday() == FIRSTWEEKDAY:
break
def get_next_month(date):
"I can't believe this isn't in the standard library!"
if date.month == 12:
return datetime.date(date.year + 1, 1, 1)
else:
return datetime.date(date.year, date.month + 1, 1)
def get_previous_month(date):
if date.month == 1:
return datetime.date(date.year - 1, 12, 1)
else:
return datetime.date(date.year, date.month - 1, 1)
@register.inclusion_tag("includes/calendar.html")
def render_calendar(date):
return calendar_context(date)
@register.inclusion_tag("includes/calendar.html")
def render_calendar_month_only(date):
ctxt = calendar_context(date)
ctxt["month_only"] = True
return ctxt
MODELS_TO_CHECK = ( # Name, model, score
("links", Blogmark, 2, "created"),
("entries", Entry, 4, "created"),
("quotes", Quotation, 2, "created"),
("photos", Photo, 1, "created"),
("photosets", Photoset, 2, "primary__created"),
)
def make_empty_day_dict(date):
d = dict([(key, []) for key, _1, _2, _3 in MODELS_TO_CHECK])
d.update({"day": date, "populated": False, "display": True})
return d
def attribute_lookup(obj, attr_string):
"Attr string is something like 'primary__created"
lookups = attr_string.split("__")
for lookup in lookups:
obj = getattr(obj, lookup)
return obj
def calendar_context(date):
"Renders a summary calendar for the given month"
day_things = dict(
[(d, make_empty_day_dict(d)) for d in itermonthdates(date.year, date.month)]
)
# Flag all days NOT in year/month as display: False
for day in list(day_things.keys()):
if day.month != date.month:
day_things[day]["display"] = False
for name, model, score, created_lookup in MODELS_TO_CHECK:
lookup_args = {
created_lookup + "__month": date.month,
created_lookup + "__year": date.year,
}
for item in model.objects.filter(**lookup_args):
day = day_things[attribute_lookup(item, created_lookup).date()]
day[name].append(item)
day["populated"] = True
# Now that we've gathered the data we can render the calendar
days = list(day_things.values())
days.sort(key=lambda x: x["day"])
# But first, swoop through and add a description to every day
for day in days:
day["score"] = score_for_day(day)
if day["populated"]:
day["description"] = description_for_day(day)
if day["day"] == date:
day["is_this_day"] = True
# Now swoop through again, applying a colour to every day based on score
cg = ColourGradient(WHITE, PURPLE)
max_score = max([d["score"] for d in days] + [0.001])
for day in days:
day["colour"] = cg.pick_css(float(day["score"]) / max_score)
weeks = []
while days:
weeks.append(days[0:7])
del days[0:7]
# Find next and previous months
# WARNING: This makes an assumption that I posted at least one thing every
# month since I started.
first_month = Entry.objects.all().order_by("created")[0].created.date()
if get_next_month(first_month) <= date:
previous_month = get_previous_month(date)
else:
previous_month = None
if date < datetime.date.today().replace(day=1):
next_month = get_next_month(date)
else:
next_month = None
return {
"next_month": next_month,
"previous_month": previous_month,
"date": date,
"weeks": weeks,
}
PURPLE = (163, 143, 183)
WHITE = (255, 255, 255)
class ColourGradient(object):
def __init__(self, min_col, max_col):
self.min_col = min_col
self.max_col = max_col
def pick(self, f):
f = float(f)
assert 0.0 <= f <= 1.0, "argument must be between 0 and 1, inclusive"
def calc(pair):
return (pair[0] - pair[1]) * f + pair[1]
return tuple(map(calc, list(zip(self.max_col, self.min_col))))
def pick_css(self, f):
"Returns e.g. rgb(0, 0, 0)"
return "rgb(%s)" % ", ".join(map(str, list(map(int, self.pick(f)))))
def description_for_day(day):
bits = []
for key in day:
if isinstance(day[key], list) and len(day[key]) > 0:
count = len(day[key])
if count == 1:
name = day[key][0]._meta.verbose_name
else:
name = day[key][0]._meta.verbose_name_plural
bits.append("%d %s" % (count, name))
return ", ".join(bits)
def score_for_day(day):
"1 point/photo, 2 points for blogmark/quote/photoset, 4 points for entry"
score = 0
for name, model, points, created_lookup in MODELS_TO_CHECK:
score += points * len(day[name])
return score
</document_content>
</document>
<document index="33">
<source>./blog/templatetags/blog_tags.py</source>
<document_content>
from django import template
register = template.Library()
@register.inclusion_tag("includes/blog_mixed_list.html", takes_context=True)
def blog_mixed_list(context, items):
context.update({"items": items, "showdate": False})
return context
@register.inclusion_tag("includes/blog_mixed_list.html", takes_context=True)
def blog_mixed_list_with_dates(
context, items, year_headers=False, day_headers=False, day_links=False
):
context.update(
{
"items": items,
"showdate": not day_headers,
"year_headers": year_headers,
"day_headers": day_headers,
"day_links": day_links,
}
)
return context
@register.inclusion_tag("includes/comments_list.html", takes_context=True)
def comments_list(context, comments):
context.update(
{
"comments": comments,
"show_headers": False,
}
)
return context
@register.inclusion_tag("includes/comments_list.html", takes_context=True)
def comments_list_with_headers(context, comments):
context.update(
{
"comments": comments,
"show_headers": True,
}
)
return context
@register.simple_tag(takes_context=True)
def page_href(context, page):
query_dict = context["request"].GET.copy()
if page == 1 and "page" in query_dict:
del query_dict["page"]
query_dict["page"] = str(page)
return "?" + query_dict.urlencode()
@register.simple_tag(takes_context=True)
def add_qsarg(context, name, value):
query_dict = context["request"].GET.copy()
if value not in query_dict.getlist(name):
query_dict.appendlist(name, value)
# And always remove ?page= - see
# https://github.com/simonw/simonwillisonblog/issues/239
if "page" in query_dict:
query_dict.pop("page")
return "?" + query_dict.urlencode()
@register.simple_tag(takes_context=True)
def remove_qsarg(context, name, value):
query_dict = context["request"].GET.copy()
query_dict.setlist(name, [v for v in query_dict.getlist(name) if v != value])
return "?" + query_dict.urlencode()
@register.simple_tag(takes_context=True)
def replace_qsarg(context, name, value):
query_dict = context["request"].GET.copy()
query_dict[name] = value
return "?" + query_dict.urlencode()
</document_content>
</document>
<document index="34">
<source>./blog/templatetags/entry_tags.py</source>
<document_content>
from django import template
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe
from xml.etree import ElementTree
import re
import datetime
register = template.Library()
entry_stripper = re.compile("^<entry>(.*?)</entry>$", re.DOTALL)
@register.filter
def xhtml(xhtml):
return XhtmlString(xhtml, contains_markup=True)
class XhtmlString(object):
def __init__(self, value, contains_markup=False):
if isinstance(value, XhtmlString):
self.et = value.et
else:
if not contains_markup:
# Handle strings like "this & that"
value = conditional_escape(value)
self.et = ElementTree.fromstring("<entry>%s</entry>" % value)
def __str__(self):
m = entry_stripper.match(ElementTree.tostring(self.et, "unicode"))
if m:
return mark_safe(m.group(1))
else:
return "" # If we end up with <entry />
@register.filter
def resize_images_to_fit_width(value, arg):
max_width = int(arg)
x = XhtmlString(value)
for img in x.et.findall(".//img"):
width = int(img.get("width", 0))
height = int(img.get("height", 0))
if width > max_width:
# Scale down
img.set("width", str(max_width))
img.set("height", str(int(float(max_width) / width * height)))
return x
xhtml_endtag_fragment = re.compile(r"\s*/>")
@register.filter
def xhtml2html(xhtml):
# ' is valid in XML/XHTML but not in regular HTML
s = str(xhtml).replace("'", "'")
return mark_safe(xhtml_endtag_fragment.sub(">", s))
@register.filter
def split_cutoff(xhtml):
return xhtml.split("<!-- cutoff -->")[0]
@register.filter
def remove_context_paragraph(xhtml):
x = XhtmlString(xhtml)
p = x.et.find("p")
if p is None:
return x
xhtml = ElementTree.tostring(p, "unicode")
if xhtml.startswith("<p><em>My answer to") or xhtml.startswith(
'<p class="context">'
):
x.et.remove(p)
return x
@register.filter
def first_paragraph(xhtml):
x = XhtmlString(xhtml)
p = x.et.find("p")
if p is not None:
return mark_safe(ElementTree.tostring(p, "unicode"))
else:
return mark_safe("<p>%s</p>" % xhtml)
@register.filter
def openid_to_url(openid):
openid = openid.strip()
if openid[0] in ("=", "@"):
return "http://xri.net/%s" % openid
else:
return openid
@register.filter
def ends_with_punctuation(value):
"""Does this end in punctuation? Use to decide if more is needed."""
last_char = value.strip()[-1]
return last_char in "?.!"
@register.filter
def strip_p_ids(xhtml):
x = XhtmlString(xhtml)
for p in x.et.findall(".//p"):
if "id" in p.attrib:
del p.attrib["id"]
return x
@register.filter
def break_up_long_words(xhtml, length):
"""Breaks up words that are longer than the argument."""
length = int(length)
x = XhtmlString(xhtml)
do_break_long_words(x.et, length)
return x
def do_break_long_words(et, length):
"""Pass an ElementTree instance; breaks up long words in it"""
if et.text:
et.text = do_break_long_words_string(et.text, length)
for child in et:
do_break_long_words(child, length)
if et.tail:
et.tail = do_break_long_words_string(et.tail, length)
whitespace_re = re.compile(r"(\s+)")
def do_break_long_words_string(s, length):
bits = whitespace_re.split(s)
for i, bit in enumerate(bits):
if whitespace_re.match(bit):
continue
if len(bit) > length:
s = ""
while bit:
s += bit[:length] + " "
bit = bit[length:]
bits[i] = s
return "".join(bits)
@register.filter
def typography(xhtml):
if not xhtml:
return xhtml
"Handles curly quotes and em dashes. Must be fed valid XHTML!"
x = XhtmlString(xhtml)
do_typography(x.et)
return x
@register.filter
def strip_wrapping_p(xhtml):
xhtml = xhtml.strip()
if xhtml.startswith("<p>"):
xhtml = xhtml[3:]
if xhtml.endswith("</p>"):
xhtml = xhtml[:-4]
return mark_safe(xhtml)
def do_typography(et):
# Designed to be called recursively on ElementTree objects
if et.tag not in ("pre", "code"):
# Don't do et.text or children for those tags; just do et.tail
if et.text:
et.text = do_typography_string(et.text)
for child in et:
do_typography(child)
if et.tail:
et.tail = do_typography_string(et.tail)
LEFT_DOUBLE_QUOTATION_MARK = "\u201c"
RIGHT_DOUBLE_QUOTATION_MARK = "\u201d"
RIGHT_SINGLE_QUOTATION_MARK = "\u2019"
QUOTATION_PAIR = (LEFT_DOUBLE_QUOTATION_MARK, RIGHT_DOUBLE_QUOTATION_MARK)
EM_DASH = "\u2014"
def quote_alternator():
i = 0
while True:
yield QUOTATION_PAIR[i % 2]
i += 1
double_re = re.compile('"')
tag_contents_re = re.compile("(<.*?>)", re.DOTALL)
def do_typography_string(s):
# Only do this on text that isn't between < and >
if "<" in s and ">" in s:
bits = tag_contents_re.split(s)
# Avoid recursion error
if len(bits) == 1:
return s
for i, bit in enumerate(bits):
if i % 2 == 0:
bits[i] = do_typography_string(bit)
return "".join(bits)
# Do single quotes
s = s.replace("'", RIGHT_SINGLE_QUOTATION_MARK)
# Now do double quotes, but only if an even number of them
if s.count('"') % 2 == 0:
alternator = quote_alternator()
s = double_re.sub(lambda m: next(alternator), s)
# Finally, do em dashes
s = s.replace(" - ", "\u2014")
return s
NUMBERS = "zero one two three four five six seven eight nine".split()
number_re = re.compile(r"\d+")
def num_to_string(s):
try:
return NUMBERS[int(s)]
except IndexError:
return s
chunks = (
(60 * 60 * 24 * 365, "year"),
(60 * 60 * 24 * 30, "month"),
(60 * 60 * 24, "day"),
(60 * 60, "hour"),
(60, "minute"),
)
@register.filter
def text_ago(d):
"""Returns 'One day' or 'Three minutes' etc - similar to time_since"""
delta = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) - d
since_seconds = (24 * 60 * 60 * delta.days) + delta.seconds
for i, (seconds, name) in enumerate(chunks):
count = since_seconds // seconds
if count != 0:
break
text = "%d %s%s" % (count, name, ((count == 1) and [""] or ["s"])[0])
# Now convert the number at the start to a text equivalent
text = number_re.sub(lambda m: num_to_string(m.group(0)), text)
# Special case for zero minutes
if text == "zero minutes" or text.startswith("-"):
text = "moments"
return "%s ago" % text
@register.inclusion_tag("includes/entry_footer.html", takes_context=True)
def entry_footer(context, entry):
context.update({"entry": entry, "showdate": True})
return context
@register.inclusion_tag("includes/entry_footer.html", takes_context=True)
def entry_footer_no_date(context, entry):
context.update({"entry": entry, "showdate": False})
return context
</document_content>
</document>
<document index="35">
<source>./blog/templatetags/tag_cloud.py</source>
<document_content>
from django import template
from django.utils.safestring import mark_safe
register = template.Library()
from blog.models import Tag
# Classes for different levels
CLASSES = (
"--skip--", # We don't show the least popular tags
"not-popular-at-all",
"not-very-popular",
"somewhat-popular",
"somewhat-more-popular",
"popular",
"more-than-just-popular",
"very-popular",
"ultra-popular",
)
def make_css_rules(
min_size=0.7, max_size=2.0, units="em", selector_prefix=".tag-cloud ."
):
num_classes = len(CLASSES)
diff_each_time = (max_size - min_size) / (num_classes - 1)
for i, klass in enumerate(CLASSES):
print(
"%s%s { font-size: %.2f%s; }"
% (selector_prefix, klass, min_size + (i * diff_each_time), units)
)
import math
def log(f):
try:
return math.log(f)
except OverflowError:
return 0
@register.inclusion_tag("includes/tag_cloud.html")
def tag_cloud_for_tags(tags):
"""
Renders a tag cloud of tags. Input should be a non-de-duped list of tag
strings.
"""
return _tag_cloud_helper(tags)
def _tag_cloud_helper(tags):
# Count them all up
tag_counts = {}
for tag in tags:
try:
tag_counts[tag] += 1
except KeyError:
tag_counts[tag] = 1
min_count = min(tag_counts.values())
max_count = max(tag_counts.values())
tags = list(tag_counts.keys())
tags.sort()
html_tags = []
intervals = 10.0
log_max = log(max_count)
log_min = log(min_count)
diff = log_max - log_min
if diff < 0.01:
# Avoid divide-by-zero problems
diff = 0.01
for tag in tags:
score = tag_counts[tag]
index = int((len(CLASSES) - 1) * (log(score) - log_min) / diff)
if CLASSES[index] == "--skip--":
continue
html_tags.append(
mark_safe(
'<a href="/tags/%s/" title="%d item%s" class="item-tag %s">%s <span>%s</span></a>'
% (tag, score, (score != 1 and "s" or ""), CLASSES[index], tag, score)
)
)
return {"tags": html_tags}
@register.inclusion_tag("includes/tag_cloud.html")
def tag_cloud():
# We do this with raw SQL for efficiency
from django.db import connection
# Get tags for entries, blogmarks, quotations
cursor = connection.cursor()
cursor.execute(
"select tag from blog_entry_tags, blog_tag where blog_entry_tags.tag_id = blog_tag.id"
)
entry_tags = [row[0] for row in cursor.fetchall()]
cursor.execute(
"select tag from blog_blogmark_tags, blog_tag where blog_blogmark_tags.tag_id = blog_tag.id"
)
blogmark_tags = [row[0] for row in cursor.fetchall()]
cursor.execute(
"select tag from blog_quotation_tags, blog_tag where blog_quotation_tags.tag_id = blog_tag.id"
)
quotation_tags = [row[0] for row in cursor.fetchall()]
cursor.close()
# Add them together
tags = entry_tags + blogmark_tags + quotation_tags
return _tag_cloud_helper(tags)
</document_content>
</document>
<document index="36">
<source>./blog/migrations/0001_initial.py</source>
<document_content>
# -*- coding: utf-8 -*-
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = []
operations = [
migrations.CreateModel(
name="Blogmark",
fields=[
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("slug", models.SlugField(max_length=64)),
("link_url", models.URLField()),
("link_title", models.CharField(max_length=255)),
("via_url", models.URLField(null=True, blank=True)),
("via_title", models.CharField(max_length=255, null=True, blank=True)),
("commentary", models.TextField()),
("created", models.DateTimeField()),
],
),
migrations.CreateModel(
name="Entry",
fields=[
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("title", models.CharField(max_length=255)),
("slug", models.SlugField(max_length=64)),
("body", models.TextField()),
("created", models.DateTimeField()),
],
),
migrations.CreateModel(
name="Photo",
fields=[
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("flickr_id", models.CharField(max_length=32)),
("server", models.CharField(max_length=8)),
("secret", models.CharField(max_length=32)),
("title", models.CharField(max_length=255, null=True, blank=True)),
("longitude", models.CharField(max_length=32, null=True, blank=True)),
("latitude", models.CharField(max_length=32, null=True, blank=True)),
("created", models.DateTimeField()),
],
),
migrations.CreateModel(
name="Photoset",
fields=[
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("flickr_id", models.CharField(max_length=32)),
("title", models.CharField(max_length=255, null=True, blank=True)),
("description", models.TextField()),
(
"photos",
models.ManyToManyField(related_name="in_photoset", to="blog.Photo"),
),
(
"primary",
models.ForeignKey(to="blog.Photo", on_delete=models.CASCADE),
),
],
),
migrations.CreateModel(
name="Quotation",
fields=[
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("slug", models.SlugField(max_length=64)),
("quotation", models.TextField()),
("source", models.CharField(max_length=255)),
("source_url", models.URLField(null=True, blank=True)),
("created", models.DateTimeField()),
],
),
migrations.CreateModel(
name="Tag",
fields=[
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("tag", models.SlugField(unique=True)),
],
),
migrations.AddField(
model_name="quotation",
name="tags",
field=models.ManyToManyField(to="blog.Tag", blank=True),
),
migrations.AddField(
model_name="entry",
name="tags",
field=models.ManyToManyField(to="blog.Tag", blank=True),
),
migrations.AddField(
model_name="blogmark",
name="tags",
field=models.ManyToManyField(to="blog.Tag", blank=True),
),
]
</document_content>
</document>
<document index="37">
<source>./blog/migrations/0002_auto_20150713_0551.py</source>
<document_content>
# -*- coding: utf-8 -*-
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
("contenttypes", "0002_remove_content_type_name"),
("blog", "0001_initial"),
]
operations = [
migrations.CreateModel(
name="Comment",
fields=[
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("object_id", models.PositiveIntegerField(db_index=True)),
("body", models.TextField()),
("created", models.DateTimeField()),
("name", models.CharField(max_length=50)),
("url", models.URLField(max_length=255, null=True, blank=True)),
("email", models.CharField(max_length=50, null=True, blank=True)),
("openid", models.CharField(max_length=255, null=True, blank=True)),
("ip", models.GenericIPAddressField()),
(
"spam_status",
models.CharField(
max_length=16,
choices=[
(b"normal", b"Not suspected"),
(b"approved", b"Approved"),
(b"suspected", b"Suspected"),
(b"spam", b"SPAM"),
],
),
),
("visible_on_site", models.BooleanField(default=True, db_index=True)),
("spam_reason", models.TextField()),
(
"content_type",
models.ForeignKey(
to="contenttypes.ContentType", on_delete=models.CASCADE
),
),
],
options={
"ordering": ["-created"],
"get_latest_by": "created",
},
),
migrations.AlterModelOptions(
name="blogmark",
options={"ordering": ("-created",)},
),
migrations.AlterModelOptions(
name="entry",
options={"ordering": ("-created",)},
),
migrations.AlterModelOptions(
name="quotation",
options={"ordering": ("-created",)},
),
]
</document_content>
</document>
<document index="38">
<source>./blog/migrations/0003_auto_20170926_0641.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-09-26 06:41
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("blog", "0002_auto_20150713_0551"),
]
operations = [
migrations.AlterModelOptions(
name="comment",
options={"get_latest_by": "created", "ordering": ("-created",)},
),
]
</document_content>
</document>
<document index="39">
<source>./blog/migrations/0004_metadata_json.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-09-29 06:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0003_auto_20170926_0641"),
]
operations = [
migrations.AddField(
model_name="blogmark",
name="metadata",
field=models.JSONField(default={}),
preserve_default=False,
),
migrations.AddField(
model_name="entry",
name="metadata",
field=models.JSONField(default={}),
preserve_default=False,
),
migrations.AddField(
model_name="quotation",
name="metadata",
field=models.JSONField(default={}),
preserve_default=False,
),
]
</document_content>
</document>
<document index="40">
<source>./blog/migrations/0005_search_document.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-09-30 20:45
import django.contrib.postgres.search
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("blog", "0004_metadata_json"),
]
operations = [
migrations.AddField(
model_name="blogmark",
name="search_document",
field=django.contrib.postgres.search.SearchVectorField(null=True),
),
migrations.AddField(
model_name="entry",
name="search_document",
field=django.contrib.postgres.search.SearchVectorField(null=True),
),
migrations.AddField(
model_name="quotation",
name="search_document",
field=django.contrib.postgres.search.SearchVectorField(null=True),
),
]
</document_content>
</document>
<document index="41">
<source>./blog/migrations/0006_gin_indexes.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-10-01 01:48
import django.contrib.postgres.indexes
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("blog", "0005_search_document"),
]
operations = [
migrations.AddIndex(
model_name="blogmark",
index=django.contrib.postgres.indexes.GinIndex(
fields=["search_document"], name="blog_blogma_search__45eeb9_gin"
),
),
migrations.AddIndex(
model_name="quotation",
index=django.contrib.postgres.indexes.GinIndex(
fields=["search_document"], name="blog_quotat_search__aa2d47_gin"
),
),
migrations.AddIndex(
model_name="entry",
index=django.contrib.postgres.indexes.GinIndex(
fields=["search_document"], name="blog_entry_search__d62c3b_gin"
),
),
]
</document_content>
</document>
<document index="42">
<source>./blog/migrations/0007_metadata_default.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-10-08 20:17
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0006_gin_indexes"),
]
operations = [
migrations.AlterField(
model_name="blogmark",
name="metadata",
field=models.JSONField(blank=True, default={}),
),
migrations.AlterField(
model_name="entry",
name="metadata",
field=models.JSONField(blank=True, default={}),
),
migrations.AlterField(
model_name="quotation",
name="metadata",
field=models.JSONField(blank=True, default={}),
),
]
</document_content>
</document>
<document index="43">
<source>./blog/migrations/0008_entry_tweet_html.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-10-08 20:35
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0007_metadata_default"),
]
operations = [
migrations.AddField(
model_name="entry",
name="tweet_html",
field=models.TextField(
blank=True,
help_text=b"Paste in the embed tweet HTML, minus the script tag,\n to display a tweet in the sidebar next to this entry.",
null=True,
),
),
]
</document_content>
</document>
<document index="44">
<source>./blog/migrations/0009_import_ref.py</source>
<document_content>
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0008_entry_tweet_html"),
]
operations = [
migrations.AddField(
model_name="blogmark",
name="import_ref",
field=models.TextField(max_length=64, null=True, unique=True),
),
migrations.AddField(
model_name="entry",
name="import_ref",
field=models.TextField(max_length=64, null=True, unique=True),
),
migrations.AddField(
model_name="quotation",
name="import_ref",
field=models.TextField(max_length=64, null=True, unique=True),
),
]
</document_content>
</document>
<document index="45">
<source>./blog/migrations/0010_auto_20180918_1646.py</source>
<document_content>
# Generated by Django 2.0b1 on 2018-09-18 16:46
import datetime
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0009_import_ref"),
]
operations = [
migrations.AlterModelOptions(
name="entry",
options={"ordering": ("-created",), "verbose_name_plural": "Entries"},
),
migrations.AlterField(
model_name="blogmark",
name="created",
field=models.DateTimeField(default=datetime.datetime.utcnow),
),
migrations.AlterField(
model_name="blogmark",
name="metadata",
field=models.JSONField(blank=True, default=dict),
),
migrations.AlterField(
model_name="comment",
name="spam_status",
field=models.CharField(
choices=[
("normal", "Not suspected"),
("approved", "Approved"),
("suspected", "Suspected"),
("spam", "SPAM"),
],
max_length=16,
),
),
migrations.AlterField(
model_name="entry",
name="created",
field=models.DateTimeField(default=datetime.datetime.utcnow),
),
migrations.AlterField(
model_name="entry",
name="metadata",
field=models.JSONField(blank=True, default=dict),
),
migrations.AlterField(
model_name="entry",
name="tweet_html",
field=models.TextField(
blank=True,
help_text="Paste in the embed tweet HTML, minus the script tag,\n to display a tweet in the sidebar next to this entry.",
null=True,
),
),
migrations.AlterField(
model_name="quotation",
name="created",
field=models.DateTimeField(default=datetime.datetime.utcnow),
),
migrations.AlterField(
model_name="quotation",
name="metadata",
field=models.JSONField(blank=True, default=dict),
),
]
</document_content>
</document>
<document index="46">
<source>./blog/migrations/0011_entry_extra_head_html.py</source>
<document_content>
# Generated by Django 2.0b1 on 2018-09-18 16:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0010_auto_20180918_1646"),
]
operations = [
migrations.AddField(
model_name="entry",
name="extra_head_html",
field=models.TextField(
blank=True,
help_text="Extra HTML to be included in the <head> for this entry",
null=True,
),
),
]
</document_content>
</document>
<document index="47">
<source>./blog/migrations/0012_card_image.py</source>
<document_content>
# Generated by Django 2.1.2 on 2019-03-07 05:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0011_entry_extra_head_html"),
]
operations = [
migrations.AddField(
model_name="blogmark",
name="card_image",
field=models.CharField(blank=True, max_length=128, null=True),
),
migrations.AddField(
model_name="entry",
name="card_image",
field=models.CharField(blank=True, max_length=128, null=True),
),
migrations.AddField(
model_name="quotation",
name="card_image",
field=models.CharField(blank=True, max_length=128, null=True),
),
]
</document_content>
</document>
<document index="48">
<source>./blog/migrations/0013_fix_simon_incutio_links.py</source>
<document_content>
from django.db import migrations
from django.utils.dates import MONTHS_3
import re
MONTHS_3_REV = {
"jan": 1,
"feb": 2,
"mar": 3,
"apr": 4,
"may": 5,
"jun": 6,
"jul": 7,
"aug": 8,
"sep": 9,
"oct": 10,
"nov": 11,
"dec": 12,
}
MONTHS_3_REV_REV = {value: key for key, value in list(MONTHS_3_REV.items())}
url_re = re.compile(
r'"http://simon\.incutio\.com/archive/(\d{4})/(\d{2})/(\d{2})/(.*?)"'
)
def fix_url(m):
yyyy, mm, dd, slug = m.groups()
month = MONTHS_3_REV_REV[int(mm)].title()
return '"/{}/{}/{}/{}/"'.format(yyyy, month, dd, slug.replace("#", ""))
def fix_simon_incutio_links(apps, schema_editor):
Entry = apps.get_model("blog", "Entry")
actually_fix_them(Entry)
def actually_fix_them(Entry):
for entry in Entry.objects.filter(body__icontains="simon.incutio"):
new_body = url_re.sub(fix_url, entry.body)
if new_body != entry.body:
Entry.objects.filter(pk=entry.pk).update(body=new_body)
path = "/%d/%s/%d/%s/" % (
entry.created.year,
MONTHS_3[entry.created.month].title(),
entry.created.day,
entry.slug,
)
print("Updated https://simonwillison.net{}".format(path))
class Migration(migrations.Migration):
dependencies = [
("blog", "0012_card_image"),
]
operations = [
migrations.RunPython(fix_simon_incutio_links),
]
</document_content>
</document>
<document index="49">
<source>./blog/migrations/0014_entry_custom_template.py</source>
<document_content>
# Generated by Django 3.1.2 on 2020-11-14 02:31
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0013_fix_simon_incutio_links"),
]
operations = [
migrations.AddField(
model_name="entry",
name="custom_template",
field=models.CharField(blank=True, max_length=100, null=True),
),
]
</document_content>
</document>
<document index="50">
<source>./blog/migrations/0015_enable_pg_trgm.py</source>
<document_content>
from django.contrib.postgres.operations import TrigramExtension
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("blog", "0014_entry_custom_template"),
]
operations = [TrigramExtension()]
</document_content>
</document>
<document index="51">
<source>./blog/migrations/0016_series.py</source>
<document_content>
import datetime
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("blog", "0015_enable_pg_trgm"),
]
operations = [
migrations.CreateModel(
name="Series",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created", models.DateTimeField(default=datetime.datetime.utcnow)),
("slug", models.SlugField(max_length=64, unique=True)),
("title", models.CharField(max_length=255)),
("summary", models.TextField()),
],
),
migrations.AddField(
model_name="blogmark",
name="series",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="blog.series",
),
),
migrations.AddField(
model_name="entry",
name="series",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="blog.series",
),
),
migrations.AddField(
model_name="quotation",
name="series",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="blog.series",
),
),
]
</document_content>
</document>
<document index="52">
<source>./blog/migrations/0017_alter_series_options.py</source>
<document_content>
# Generated by Django 3.2.3 on 2021-08-07 22:34
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("blog", "0016_series"),
]
operations = [
migrations.AlterModelOptions(
name="series",
options={"verbose_name_plural": "Series"},
),
]
</document_content>
</document>
<document index="53">
<source>./blog/migrations/0018_alter_blogmark_link_url_alter_blogmark_via_url.py</source>
<document_content>
# Generated by Django 4.1.7 on 2023-05-22 19:57
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0017_alter_series_options"),
]
operations = [
migrations.AlterField(
model_name="blogmark",
name="link_url",
field=models.URLField(max_length=512),
),
migrations.AlterField(
model_name="blogmark",
name="via_url",
field=models.URLField(blank=True, max_length=512, null=True),
),
]
</document_content>
</document>
<document index="54">
<source>./blog/migrations/0019_blogmark_use_markdown.py</source>
<document_content>
# Generated by Django 5.0.2 on 2024-04-25 03:52
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0018_alter_blogmark_link_url_alter_blogmark_via_url"),
]
operations = [
migrations.AddField(
model_name="blogmark",
name="use_markdown",
field=models.BooleanField(default=False),
),
]
</document_content>
</document>
<document index="55">
<source>./blog/migrations/0020_tag_description.py</source>
<document_content>
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0019_blogmark_use_markdown"),
]
operations = [
migrations.AddField(
model_name="tag",
name="description",
field=models.TextField(blank=True),
),
]
</document_content>
</document>
<document index="56">
<source>./blog/migrations/0021_previous_tag_name.py</source>
<document_content>
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0020_tag_description"),
]
operations = [
migrations.CreateModel(
name="PreviousTagName",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("previous_name", models.SlugField()),
("tag", models.ForeignKey(on_delete=models.CASCADE, to="blog.Tag")),
],
),
]
</document_content>
</document>
<document index="57">
<source>./blog/migrations/0022_alter_blogmark_use_markdown.py</source>
<document_content>
# Generated by Django 5.1 on 2024-08-13 22:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0021_previous_tag_name"),
]
operations = [
migrations.AlterField(
model_name="blogmark",
name="use_markdown",
field=models.BooleanField(
default=False,
help_text='Images can use the img element - set width="..." for a specific width and use class="blogmark-image" to center and add a drop shadow.',
),
),
]
</document_content>
</document>
<document index="58">
<source>./blog/migrations/0023_blogmark_is_draft_entry_is_draft_quotation_is_draft.py</source>
<document_content>
# Generated by Django 5.1 on 2024-09-18 05:47
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0022_alter_blogmark_use_markdown"),
]
operations = [
migrations.AddField(
model_name="blogmark",
name="is_draft",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="entry",
name="is_draft",
field=models.BooleanField(default=False),
),
migrations.AddField(
model_name="quotation",
name="is_draft",
field=models.BooleanField(default=False),
),
]
</document_content>
</document>
<document index="59">
<source>./blog/migrations/0024_liveupdate.py</source>
<document_content>
# Generated by Django 5.1.1 on 2024-10-01 16:57
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0023_blogmark_is_draft_entry_is_draft_quotation_is_draft"),
]
operations = [
migrations.CreateModel(
name="LiveUpdate",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created", models.DateTimeField(auto_now_add=True)),
("content", models.TextField()),
(
"entry",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="updates",
to="blog.entry",
),
),
],
),
]
</document_content>
</document>
<document index="60">
<source>./blog/migrations/0025_entry_live_timezone.py</source>
<document_content>
# Generated by Django 5.1.1 on 2024-10-01 17:02
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0024_liveupdate"),
]
operations = [
migrations.AddField(
model_name="entry",
name="live_timezone",
field=models.CharField(blank=True, max_length=100, null=True),
),
]
</document_content>
</document>
<document index="61">
<source>./blog/migrations/0026_quotation_context.py</source>
<document_content>
# Generated by Django 5.1.1 on 2024-10-11 12:51
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0025_entry_live_timezone"),
]
operations = [
migrations.AddField(
model_name="quotation",
name="context",
field=models.CharField(blank=True, max_length=255, null=True),
),
]
</document_content>
</document>
<document index="62">
<source>./blog/migrations/0027_add_tag_type.py</source>
<document_content>
# Generated by Django 5.1.1 on 2025-03-09 17:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("blog", "0026_quotation_context"),
]
operations = [
migrations.AddField(
model_name="tag",
name="tag_type",
field=models.CharField(blank=True, max_length=32, null=True),
),
]
</document_content>
</document>
<document index="63">
<source>./blog/migrations/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="64">
<source>./blog/management/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="65">
<source>./blog/management/commands/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="66">
<source>./blog/management/commands/import_blog_json.py</source>
<document_content>
from django.core.management.base import BaseCommand
from datetime import timezone
from blog.models import (
Entry,
Blogmark,
Tag,
Quotation,
)
import requests
from dateutil import parser
import json
class Command(BaseCommand):
help = """
./manage.py import_blog_json URL-or-path-to-JSON
"""
def add_arguments(self, parser):
parser.add_argument(
"url_or_path_to_json",
type=str,
help="URL or path to JSON to import",
)
parser.add_argument(
"--tag_with",
action="store",
dest="tag_with",
default=False,
help="Tag to apply to all imported items",
)
def handle(self, *args, **kwargs):
url_or_path_to_json = kwargs["url_or_path_to_json"]
tag_with = kwargs["tag_with"]
tag_with_tag = None
if tag_with:
tag_with_tag = Tag.objects.get_or_create(tag=tag_with)[0]
is_url = url_or_path_to_json.startswith(
"http://"
) or url_or_path_to_json.startswith("https://")
if is_url:
items = requests.get(url_or_path_to_json).json()
else:
items = json.load(open(url_or_path_to_json))
for item in items:
created = parser.parse(item["datetime"]).replace(tzinfo=timezone.utc)
was_created = False
slug = item["slug"][:64].strip("-")
if item["type"] == "entry":
klass = Entry
kwargs = dict(
body=item["body"],
title=item["title"],
created=created,
slug=slug,
metadata=item,
)
elif item["type"] == "quotation":
klass = Quotation
kwargs = dict(
quotation=item["quotation"],
source=item["source"],
source_url=item["source_url"],
created=created,
slug=slug,
metadata=item,
)
elif item["type"] == "blogmark":
klass = Blogmark
kwargs = dict(
slug=slug,
link_url=item["link_url"],
link_title=item["link_title"],
via_url=item.get("via_url") or "",
via_title=item.get("via_title") or "",
commentary=item["commentary"] or "",
created=created,
metadata=item,
)
else:
assert False, "type should be known, %s" % item["type"]
if item.get("import_ref"):
obj, was_created = klass.objects.update_or_create(
import_ref=item["import_ref"], defaults=kwargs
)
else:
obj = klass.objects.create(**kwargs)
tags = [Tag.objects.get_or_create(tag=tag)[0] for tag in item["tags"]]
if tag_with_tag:
tags.append(tag_with_tag)
obj.tags.set(tags)
print(was_created, obj, obj.get_absolute_url())
</document_content>
</document>
<document index="67">
<source>./blog/management/commands/import_blog_xml.py</source>
<document_content>
from django.conf import settings
from django.core.management.base import BaseCommand
from django.contrib.contenttypes.models import ContentType
from blog.models import (
Entry,
Blogmark,
Tag,
Quotation,
Comment,
)
from xml.etree import ElementTree as ET
import os
import sys
def iter_rows(filepath):
# Iterate over rows in a SequelPro XML dump
et = ET.parse(open(filepath))
for row in et.findall("database/table_data/row"):
d = {}
for field in row.findall("field"):
d[field.attrib["name"]] = field.text
yield d
class Command(BaseCommand):
help = """
./manage.py import_blog_xml
"""
def add_arguments(self, parser):
parser.add_argument(
"--xmldir",
action="store",
dest="xmldir",
default=os.path.join(settings.BASE_DIR, "old-import-xml"),
help="Directory where the XML files live",
)
def handle(self, *args, **kwargs):
xmldir = kwargs["xmldir"]
import_tags(xmldir)
import_entries(xmldir)
import_blogmarks(xmldir)
import_quotations(xmldir)
import_comments(xmldir)
def import_tags(xmldir):
# First create tags
for row in iter_rows(os.path.join(xmldir, "blog_tag.xml")):
Tag.objects.get_or_create(tag=row["tag"])
def import_entries(xmldir):
# Now do entries
for row in iter_rows(os.path.join(xmldir, "blog_entry.xml")):
entry, created = Entry.objects.get_or_create(
id=row["id"],
defaults=dict(
body=row["body"],
created=row["created"],
title=row["title"],
slug=row["slug"],
),
)
print(entry, created)
# Now associate entries with tags
for row in iter_rows(os.path.join(xmldir, "blog_entry_tags.xml")):
entry_id = row["entry_id"]
tag = row["tag_id"] # actually a tag
entry = Entry.objects.get(pk=entry_id)
entry.tags.add(Tag.objects.get(tag=tag))
def import_blogmarks(xmldir):
# Next do blogmarks
for row in iter_rows(os.path.join(xmldir, "blog_blogmark.xml")):
blogmark, created = Blogmark.objects.get_or_create(
id=row["id"],
defaults=dict(
slug=row["slug"],
link_url=row["link_url"],
link_title=row["link_title"],
via_url=row["via_url"],
via_title=row["via_title"],
commentary=row["commentary"] or "",
created=row["created"],
),
)
for row in iter_rows(os.path.join(xmldir, "blog_blogmark_tags.xml")):
blogmark_id = row["blogmark_id"]
tag = row["tag_id"] # actually a tag
entry = Blogmark.objects.get(pk=blogmark_id)
entry.tags.add(Tag.objects.get(tag=tag))
def import_quotations(xmldir):
# and now quotations
for row in iter_rows(os.path.join(xmldir, "blog_quotation.xml")):
quotation, created = Quotation.objects.get_or_create(
id=row["id"],
defaults=dict(
slug=row["slug"],
quotation=row["quotation"],
source=row["source"],
source_url=row["source_url"],
created=row["created"],
),
)
for row in iter_rows(os.path.join(xmldir, "blog_quotation_tags.xml")):
quotation_id = row["quotation_id"]
tag = row["tag_id"] # actually a tag
entry = Quotation.objects.get(pk=quotation_id)
entry.tags.add(Tag.objects.get(tag=tag))
def import_comments(xmldir):
# Finally... comments!
# First we need to know what the old content_type IDs
# should map to
content_types_by_id = {}
for row in iter_rows(os.path.join(xmldir, "django_content_type.xml")):
content_types_by_id[row["id"]] = row
content_type_models_by_name = {}
for ct in ContentType.objects.filter(app_label="blog"):
content_type_models_by_name[ct.model] = ct
i = 0
for row in iter_rows(os.path.join(xmldir, "blog_comment.xml")):
#
# <row>
# <field name="id">31819</field>
# <field name="content_type_id">19</field>
# <field name="object_id">1503</field>
# <field name="body">http://videos.pass.as/index.html
# http://www.full-length-movies.biz/index.html
# http://download-movies.fw.nu/index.html
# http://movies.isthebe.st/index.html</field>
# <field name="created">2005-10-11 21:29:24</field>
# <field name="name">xxx</field>
# <field name="url">http://movies.isthebe.st/index.html</field>
# <field name="email">-ana@ma-.com</field>
# <field name="openid" xsi:nil="true" />
# <field name="ip">80.82.59.156</field>
# <field name="spam_status">spam</field>
# <field name="visible_on_site">0</field>
# <field name="spam_reason" xsi:nil="true" />
# </row>
Comment.objects.get_or_create(
id=row["id"],
defaults=dict(
content_type=content_type_models_by_name[
content_types_by_id[row["content_type_id"]]["model"]
],
object_id=row["object_id"],
body=row["body"],
created=row["created"] + "Z",
name=row["name"] or "",
url=row["url"],
email=row["email"],
openid=row["openid"],
ip=(row["ip"] or "0.0.0.0")
.replace("xx.xx.xx.xx", "0.0.0.0")
.replace("xxx.xxx.xxx.xxx", "0.0.0.0")
.replace("unknown", "0.0.0.0"),
spam_status=row["spam_status"],
visible_on_site=row["visible_on_site"],
spam_reason=row["spam_reason"] or "",
),
)
i += 1
if i % 100 == 0:
print(i)
sys.stdout.flush()
</document_content>
</document>
<document index="68">
<source>./blog/management/commands/import_quora.py</source>
<document_content>
from django.core.management.base import BaseCommand
from datetime import timezone
from django.db import transaction
from blog.models import (
Entry,
Tag,
)
from BeautifulSoup import BeautifulSoup as Soup
import requests
from django.utils.html import escape
from django.utils.text import slugify
from dateutil import parser
import datetime
import random
class Command(BaseCommand):
help = "./manage.py import_quora http://URL-to-JSON.json http://URL-to-topic-CSV"
def add_arguments(self, parser):
parser.add_argument("json_url", type=str)
parser.add_argument("topic_csv_url", type=str)
def handle(self, *args, **kwargs):
data_url = kwargs["json_url"]
topic_csv_url = kwargs["topic_csv_url"]
lines = requests.get(topic_csv_url).content.split("\n")
quora_to_tag = {
line.split("\t")[0]: line.split("\t")[-1].strip()
for line in lines
if line.strip()
}
posts = requests.get(data_url).json()
with transaction.atomic():
quora = Tag.objects.get_or_create(tag="quora")[0]
for post in posts:
question = post["originalQuestion"] or post["question"]
url = "https://www.quora.com" + (
post["originalQuestionUrl"] or post["href"]
)
if question.endswith("Remove Banner"):
question = question.replace("Remove Banner", "")
answer = clean_answer(post["answer"])
date = datetime.datetime.combine(
parser.parse(post["meta"].replace("Added ", "")).date(),
datetime.time(random.randint(9, 18), random.randint(0, 59)),
).replace(tzinfo=timezone.utc)
truncated_question = question
if len(truncated_question) > 250:
truncated_question = truncated_question[:250] + "..."
body = '<p><em>My answer to <a href="%s">%s</a> on Quora</em></p>' % (
url,
escape(question),
)
body += "\n\n" + answer
body = body.replace(" ", " ")
slug = slugify(" ".join(truncated_question.split()[:4]))
with transaction.atomic():
entry = Entry.objects.create(
slug=slug,
created=date,
title=truncated_question,
body=body,
metadata=post,
)
entry.tags.add(quora)
for topic in post["topics"]:
tag = quora_to_tag.get(topic)
if tag:
entry.tags.add(Tag.objects.get_or_create(tag=tag)[0])
print(entry)
def clean_answer(html):
soup = Soup(html)
# Ditch class attributes
for tag in ("p", "span", "a", "code", "div"):
for el in soup.findAll(tag):
del el["class"]
# On links, kill the rel and target and onclick and tooltip
for el in soup.findAll("a"):
del el["rel"]
del el["target"]
del el["onclick"]
del el["data-qt-tooltip"]
for el in soup.findAll("canvas"):
el.extract()
for img in soup.findAll("img"):
del img["class"]
del img["data-src"]
src = img["master_src"]
del img["master_src"]
w = img["master_w"]
del img["master_w"]
h = img["master_h"]
del img["master_h"]
img["src"] = src
img["width"] = w
img["height"] = h
img["style"] = "max-width: 100%"
# Cleanup YouTube videos
for div in soup.findAll("div", {"data-video-provider": "youtube"}):
iframe = Soup(div["data-embed"]).find("iframe")
src = "https:%s" % iframe["src"].split("?")[0]
div.replaceWith(
Soup(
"""
<iframe width="560" height="315"
src="%s" frameborder="0" allowfullscreen>
</iframe>
"""
% src
)
)
html = str(soup)
html = html.replace('<a href="/', '<a href="https://www.quora.com/')
# Replace <br /><br /> with paragraphs
chunks = html.split("<br /><br />")
new_chunks = []
for chunk in chunks:
if not chunk.startswith("<"):
chunk = "<p>%s</p>" % chunk
new_chunks.append(chunk)
return "\n\n".join(new_chunks)
</document_content>
</document>
<document index="69">
<source>./blog/management/commands/reindex_all.py</source>
<document_content>
from django.core.management.base import BaseCommand
from blog.models import Entry, Blogmark, Quotation
class Command(BaseCommand):
help = "Re-indexes all entries, blogmarks, quotations"
def handle(self, *args, **kwargs):
for klass in (Entry, Blogmark, Quotation):
i = 0
for obj in klass.objects.prefetch_related("tags").all():
obj.save()
i += 1
if i % 100 == 0:
print(klass, i)
</document_content>
</document>
<document index="70">
<source>./blog/management/commands/validate_entries_xml.py</source>
<document_content>
from django.core.management.base import BaseCommand
from xml.etree import ElementTree
from blog.models import Entry
class Command(BaseCommand):
help = "Spits out list of entries with invalid XML"
def handle(self, *args, **kwargs):
for entry in Entry.objects.all():
try:
ElementTree.fromstring("<entry>%s</entry>" % entry.body.encode("utf8"))
except Exception as e:
print(e)
print(entry.title)
print("https://simonwillison.com/admin/blog/entry/%d/" % entry.pk)
print()
</document_content>
</document>
<document index="71">
<source>./redirects/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="72">
<source>./redirects/admin.py</source>
<document_content>
from django.contrib import admin
from .models import Redirect
admin.site.register(
Redirect, list_display=("domain", "path", "target"), list_filter=("domain",)
)
</document_content>
</document>
<document index="73">
<source>./redirects/apps.py</source>
<document_content>
# -*- coding: utf-8 -*-
from django.apps import AppConfig
class RedirectsConfig(AppConfig):
name = "redirects"
</document_content>
</document>
<document index="74">
<source>./redirects/middleware.py</source>
<document_content>
from .models import Redirect
from django.http import HttpResponsePermanentRedirect
def redirect_middleware(get_response):
def middleware(request):
path = request.path.lstrip("/")
redirects = list(
Redirect.objects.filter(
domain=request.get_host(),
# We redirect on either a path match or a '*'
# record existing for this domain
path__in=(path, "*"),
)
)
# A non-star redirect always takes precedence
non_star = [r for r in redirects if r.path != "*"]
if non_star:
return HttpResponsePermanentRedirect(non_star[0].target)
# If there's a star redirect, build path and redirect to that
star = [r for r in redirects if r.path == "*"]
if star:
new_url = star[0].target + path
if request.META["QUERY_STRING"]:
new_url += "?" + request.META["QUERY_STRING"]
return HttpResponsePermanentRedirect(new_url)
# Default: no redirects, just get on with it:
return get_response(request)
return middleware
</document_content>
</document>
<document index="75">
<source>./redirects/models.py</source>
<document_content>
from django.db import models
class Redirect(models.Model):
domain = models.CharField(max_length=128, blank=True)
path = models.CharField(max_length=128, blank=True)
target = models.CharField(max_length=256, blank=True)
created = models.DateTimeField(auto_now_add=True)
class Meta:
constraints = [
models.UniqueConstraint(fields=["domain", "path"], name="unique_redirect")
]
def __unicode__(self):
return "%s/%s => %s" % (self.domain, self.path, self.target)
</document_content>
</document>
<document index="76">
<source>./redirects/tests.py</source>
<document_content>
# -*- coding: utf-8 -*-
from django.test import TestCase
# Create your tests here.
</document_content>
</document>
<document index="77">
<source>./redirects/views.py</source>
<document_content>
# -*- coding: utf-8 -*-
from django.shortcuts import render
# Create your views here.
</document_content>
</document>
<document index="78">
<source>./redirects/migrations/0001_initial.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-10-01 20:58
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Redirect",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("domain", models.CharField(blank=True, max_length=128, null=True)),
("path", models.CharField(blank=True, max_length=128, null=True)),
("target", models.CharField(blank=True, max_length=256, null=True)),
("created", models.DateTimeField(auto_now_add=True)),
],
),
# migrations.AlterUniqueTogether(
# name="redirect",
# unique_together=set([("domain", "path")]),
# ),
]
</document_content>
</document>
<document index="79">
<source>./redirects/migrations/0002_auto_20171001_2242.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-10-01 22:42
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("redirects", "0001_initial"),
]
operations = [
migrations.AlterField(
model_name="redirect",
name="domain",
field=models.CharField(blank=True, default="", max_length=128),
preserve_default=False,
),
migrations.AlterField(
model_name="redirect",
name="path",
field=models.CharField(blank=True, default="", max_length=128),
preserve_default=False,
),
migrations.AlterField(
model_name="redirect",
name="target",
field=models.CharField(blank=True, default="", max_length=256),
preserve_default=False,
),
]
</document_content>
</document>
<document index="80">
<source>./redirects/migrations/0003_alter_redirect_unique_together_and_more.py</source>
<document_content>
# Generated by Django 5.1 on 2024-08-13 22:51
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("redirects", "0002_auto_20171001_2242"),
]
operations = [
migrations.AlterUniqueTogether(
name="redirect",
unique_together=set(),
),
migrations.AddConstraint(
model_name="redirect",
constraint=models.UniqueConstraint(
fields=("domain", "path"), name="unique_redirect"
),
),
]
</document_content>
</document>
<document index="81">
<source>./redirects/migrations/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="82">
<source>./feedstats/__init__.py</source>
<document_content>
</document_content>
</document>
<document index="83">
<source>./feedstats/admin.py</source>
<document_content>
from django.contrib import admin
from .models import SubscriberCount
admin.site.register(
SubscriberCount,
list_display=("path", "user_agent", "count", "created"),
list_filter=("path", "created"),
)
</document_content>
</document>
<document index="84">
<source>./feedstats/apps.py</source>
<document_content>
from django.apps import AppConfig
class FeedstatsConfig(AppConfig):
name = "feedstats"
</document_content>
</document>
<document index="85">
<source>./feedstats/models.py</source>
<document_content>
from django.db import models
class SubscriberCount(models.Model):
path = models.CharField(max_length=128)
count = models.IntegerField()
created = models.DateTimeField(auto_now_add=True)
user_agent = models.CharField(max_length=256, db_index=True)
class Meta:
constraints = [
models.UniqueConstraint(
fields=["path", "user_agent", "count", "created"],
name="unique_subscriber_count",
)
]
</document_content>
</document>
<document index="86">
<source>./feedstats/tests.py</source>
<document_content>
from django.test import TestCase
from .models import SubscriberCount
import datetime
class FeedstatsTests(TestCase):
def test_feedstats_records_subscriber_numbers(self):
self.assertEqual(0, SubscriberCount.objects.count())
# If no \d+ subscribers, we don't record anything
self.client.get("/atom/everything/", HTTP_USER_AGENT="Blah")
self.assertEqual(0, SubscriberCount.objects.count())
self.client.get("/atom/everything/", HTTP_USER_AGENT="Blah (10 subscribers)")
self.assertEqual(1, SubscriberCount.objects.count())
row = SubscriberCount.objects.all()[0]
self.assertEqual("/atom/everything/", row.path)
self.assertEqual(10, row.count)
self.assertEqual(datetime.date.today(), row.created.date())
self.assertEqual("Blah (X subscribers)", row.user_agent)
# If we hit again with the same number, no new record is recorded
self.client.get("/atom/everything/", HTTP_USER_AGENT="Blah (10 subscribers)")
self.assertEqual(1, SubscriberCount.objects.count())
# If we hit again with a different number, we record a new row
self.client.get("/atom/everything/", HTTP_USER_AGENT="Blah (11 subscribers)")
self.assertEqual(2, SubscriberCount.objects.count())
row = SubscriberCount.objects.all()[1]
self.assertEqual(11, row.count)
self.assertEqual("Blah (X subscribers)", row.user_agent)
</document_content>
</document>
<document index="87">
<source>./feedstats/utils.py</source>
<document_content>
from functools import wraps
from .models import SubscriberCount
import datetime
import re
subscribers_re = re.compile(r"(\d+) subscribers")
def count_subscribers(view_fn):
@wraps(view_fn)
def inner_fn(request, *args, **kwargs):
user_agent = request.META.get("HTTP_USER_AGENT", "")
match = subscribers_re.search(user_agent)
if match:
count = int(match.group(1))
today = datetime.date.today()
simplified_user_agent = subscribers_re.sub("X subscribers", user_agent)
# Do we have this one yet?
if not SubscriberCount.objects.filter(
path=request.path,
count=count,
user_agent=simplified_user_agent,
created__year=today.year,
created__month=today.month,
created__day=today.day,
).exists():
SubscriberCount.objects.create(
path=request.path,
count=count,
user_agent=simplified_user_agent,
)
return view_fn(request, *args, **kwargs)
return inner_fn
</document_content>
</document>
<document index="88">
<source>./feedstats/migrations/0001_initial.py</source>
<document_content>
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="SubscriberCount",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("path", models.CharField(max_length=128)),
("count", models.IntegerField()),
("created", models.DateTimeField(auto_now_add=True)),
("user_agent", models.CharField(db_index=True, max_length=128)),
],
),
# migrations.AlterIndexTogether(
# name="subscribercount",
# index_together=set([("path", "user_agent", "count", "created")]),
# ),
]
</document_content>
</document>
<document index="89">
<source>./feedstats/migrations/0002_longer_user_agent_field.py</source>
<document_content>
# -*- coding: utf-8 -*-
# Generated by Django 1.11.5 on 2017-10-20 18:52
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("feedstats", "0001_initial"),
]
operations = [
migrations.AlterField(
model_name="subscribercount",
name="user_agent",
field=models.CharField(db_index=True, max_length=256),
),
]
</document_content>
</document>
<document index="90">
<source>./feedstats/migrations/0003_subscribercount_unique_subscriber_count.py</source>
<document_content>
# Generated by Django 5.1 on 2024-08-13 22:50
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("feedstats", "0002_longer_user_agent_field"),
]
operations = [
migrations.AddConstraint(
model_name="subscribercount",
constraint=models.UniqueConstraint(
fields=("path", "user_agent", "count", "created"),
name="unique_subscriber_count",
),
),
]
</document_content>
</document>
<document index="91">
<source>./feedstats/migrations/__init__.py</source>
<document_content>
</document_content>
</document>
</documents>
I want to add a new type of content called a Note, similar to quotation and bookmark and entry but it only has a markdown text body. Output all of the code I need to add for that feature and tell me which files to add the code to.
Okay, let's add the Note content type. This involves creating the model, integrating it into the admin, views, templates, feeds, search, and other relevant parts of the application.
Here is the code you need to add and the files to put it in:
1. Add the Note model:
- File:
blog/models.py - Action: Add the
Noteclass definition and updateload_mixed_objects.
# blog/models.py
# ... (import markdown, etc. near the top)
from markdown import markdown
# ... (other model definitions like Entry, Quotation, Blogmark)
class Note(BaseModel):
body = models.TextField()
is_note = True
def body_rendered(self):
return mark_safe(markdown(self.body))
def index_components(self):
# Note: 'A' is typically title/headline, 'C' is main body, 'B' is tags
return {
"C": self.body,
"B": " ".join(self.tags.values_list("tag", flat=True)),
}
def __str__(self):
# Return first 50 chars as string representation
if len(self.body) > 50:
return self.body[:50] + "..."
return self.body
class Meta(BaseModel.Meta):
verbose_name_plural = "Notes"
# ... (rest of the file, including the load_mixed_objects function)
def load_mixed_objects(dicts):
"""
Takes a list of dictionaries, each of which must at least have a 'type'
and a 'pk' key. Returns a list of ORM objects of those various types.
Each returned ORM object has a .original_dict attribute populated.
"""
to_fetch = {}
for d in dicts:
to_fetch.setdefault(d["type"], set()).add(d["pk"])
fetched = {}
# *** ADD 'note': Note HERE ***
for key, model in (
("blogmark", Blogmark),
("entry", Entry),
("quotation", Quotation),
("note", Note), # <-- Add this line
):
ids = to_fetch.get(key) or []
objects = model.objects.prefetch_related("tags").filter(pk__in=ids)
for obj in objects:
fetched[(key, obj.pk)] = obj
# Build list in same order as dicts argument
to_return = []
for d in dicts:
item = fetched.get((d["type"], d["pk"])) or None
if item:
item.original_dict = d
to_return.append(item)
return to_return
# *** Update Tag.total_count and Tag.all_types_queryset ***
class Tag(models.Model):
# ... (existing fields and methods)
def note_count(self):
return self.note_set.filter(is_draft=False).count()
def total_count(self):
entry_count = Subquery(
Entry.objects.filter(is_draft=False, tags=OuterRef("pk"))
.values("tags")
.annotate(count=Count("id"))
.values("count"),
output_field=IntegerField(),
)
blogmark_count = Subquery(
Blogmark.objects.filter(is_draft=False, tags=OuterRef("pk"))
.values("tags")
.annotate(count=Count("id"))
.values("count"),
output_field=IntegerField(),
)
quotation_count = Subquery(
Quotation.objects.filter(is_draft=False, tags=OuterRef("pk"))
.values("tags")
.annotate(count=Count("id"))
.values("count"),
output_field=IntegerField(),
)
# *** ADD Note count subquery ***
note_count = Subquery(
Note.objects.filter(is_draft=False, tags=OuterRef("pk"))
.values("tags")
.annotate(count=Count("id"))
.values("count"),
output_field=IntegerField(),
)
result = (
Tag.objects.filter(pk=self.pk)
.annotate(
total_count=Sum(
Coalesce(entry_count, 0)
+ Coalesce(blogmark_count, 0)
+ Coalesce(quotation_count, 0)
+ Coalesce(note_count, 0) # <-- Add this
)
)
.values("total_count")
.first()
)
return result["total_count"] if result else 0
def all_types_queryset(self):
entries = (
self.entry_set.all()
.annotate(type=models.Value("entry", output_field=models.CharField()))
.values("pk", "created", "type")
)
blogmarks = (
self.blogmark_set.all()
.annotate(type=models.Value("blogmark", output_field=models.CharField()))
.values("pk", "created", "type")
)
quotations = (
self.quotation_set.all()
.annotate(type=models.Value("quotation", output_field=models.CharField()))
.values("pk", "created", "type")
)
# *** ADD Note queryset ***
notes = (
self.note_set.all()
.annotate(type=models.Value("note", output_field=models.CharField()))
.values("pk", "created", "type")
)
# *** ADD notes to the union ***
return entries.union(blogmarks, quotations, notes).order_by("-created") # <-- Add notes here
def get_related_tags(self, limit=10):
"""Get all items tagged with this, look at /their/ tags, order by count"""
if not hasattr(self, "_related_tags"):
counts = Counter()
# *** ADD Note to the list of classes ***
for klass, collection in (
(Entry, "entry_set"),
(Blogmark, "blogmark_set"),
(Quotation, "quotation_set"),
(Note, "note_set"), # <-- Add this line
):
qs = klass.objects.filter(
pk__in=getattr(self, collection).all()
).values_list("tags__tag", flat=True)
counts.update(t for t in qs if t != self.tag)
tag_names = [p[0] for p in counts.most_common(limit)]
tags_by_name = {t.tag: t for t in Tag.objects.filter(tag__in=tag_names)}
# Need a list in the correct order
self._related_tags = [tags_by_name[name] for name in tag_names]
return self._related_tags
# ... (rest of Tag model)
# *** Update COMMENTS_ALLOWED_ON ***
COMMENTS_ALLOWED_ON = ("entry", "blogmark", "quotation", "note") # <-- Add 'note'2. Add the Note model to the Admin:
- File:
blog/admin.py - Action: Import
Noteand register it with a newNoteAdminclass.
# blog/admin.py
from .models import (
Entry,
Tag,
Quotation,
Blogmark,
Comment,
Series,
PreviousTagName,
LiveUpdate,
Note, # <-- Add Note here
)
# ... (BaseAdmin definition)
@admin.register(Entry)
class EntryAdmin(BaseAdmin):
# ... (existing EntryAdmin content)
# ... (Other Admin classes)
# *** ADD NoteAdmin HERE ***
@admin.register(Note)
class NoteAdmin(BaseAdmin):
search_fields = ("tags__tag", "body")
list_display = ("__str__", "created", "tag_summary")
# ... (rest of the file)3. Create Database Migration:
- Action: Run the following command in your terminal:
python manage.py makemigrations blog
- Result: This will create a new migration file in
blog/migrations/(e.g.,0028_note.py). This file contains the instructions to create theblog_notetable and its relationship toblog_tag.
4. Apply the Migration:
- Action: Run the following command in your terminal:
python manage.py migrate
5. Update Views to Handle Notes:
- File:
blog/views.py - Action: Modify
archive_item,index,archive_month,archive_day,archive_tag,find_current_tags, and addredirect_note.
# blog/views.py
from .models import (
Blogmark,
Entry,
Quotation,
Note, # <-- Add Note
Photo,
# ... other models
)
# ... (other imports)
def archive_item(request, year, month, day, slug):
# ... (existing day parsing logic)
# *** ADD 'note': Note to the list ***
for content_type, model in (
("blogmark", Blogmark),
("entry", Entry),
("quotation", Quotation),
("note", Note), # <-- Add this line
):
try:
obj = get_object_or_404(
model,
# ... (existing filters)
)
except Http404:
continue
# ... (existing previously_hosted logic, should not apply to Note)
if content_type == "entry": # Keep this check specific to entry
# ... existing previously_hosted logic for entries ...
pass
else:
previously_hosted = None
template = getattr(obj, "custom_template", None) or "{}.html".format(
content_type
)
# ... (existing updates logic for Entry)
response = render(
request,
template,
{
content_type: obj, # Will be 'note' for notes
"content_type": content_type,
"object_id": obj.id,
"previously_hosted": previously_hosted,
"item": obj,
"recent_articles": Entry.objects.filter(is_draft=False)
.prefetch_related("tags")
.order_by("-created")[0:3],
"is_draft": obj.is_draft,
# Only include updates for entries
"updates": updates if isinstance(obj, Entry) else [],
},
)
if obj.is_draft:
set_no_cache(response)
return response
# If we get here, none of the views matched
raise Http404
def index(request):
# Get back 30 most recent across all item types
recent = list(
Entry.objects.filter(is_draft=False)
# ... (existing Entry union part)
.union(
Blogmark.objects.filter(is_draft=False)
# ... (existing Blogmark union part)
)
.union(
Quotation.objects.filter(is_draft=False)
# ... (existing Quotation union part)
)
# *** ADD Note union part HERE ***
.union(
Note.objects.filter(is_draft=False)
.annotate(content_type=Value("note", output_field=CharField()))
.values("content_type", "id", "created")
.order_by()
)
.order_by("-created")[:30]
)
# Now load the objects
items = []
to_load = {}
for item in recent:
to_load.setdefault(item["content_type"], []).append(item["id"])
# *** ADD 'note': Note HERE ***
for content_type, model in (
("entry", Entry),
("blogmark", Blogmark),
("quotation", Quotation),
("note", Note), # <-- Add this line
):
if content_type not in to_load:
continue
objects = model.objects.prefetch_related("tags").in_bulk(to_load[content_type])
items.extend([{"type": content_type, "obj": obj} for obj in objects.values()])
items.sort(key=lambda x: x["obj"].created, reverse=True)
# ... (rest of index view)
def find_current_tags(num=5):
"""Returns num random tags from top 30 in recent 400 taggings"""
last_400_tags = list(
Tag.quotation_set.through.objects.annotate(
created=models.F("quotation__created")
)
.values("tag__tag", "created")
.union(
Tag.entry_set.through.objects.annotate(
created=models.F("entry__created")
).values("tag__tag", "created"),
Tag.blogmark_set.through.objects.annotate(
created=models.F("blogmark__created")
).values("tag__tag", "created"),
# *** ADD Note tags union part HERE ***
Tag.note_set.through.objects.annotate(
created=models.F("note__created")
).values("tag__tag", "created"), # <-- Add this
)
.order_by("-created")[:400]
)
# ... (rest of find_current_tags)
def archive_year(request, year):
# ... (inside the loop for month in range(1, 12 + 1): )
# ... (entry_count, link_count, quote_count)
photo_count = Photo.objects.filter(
created__year=year, created__month=month
).count()
# *** ADD note_count HERE ***
note_count = Note.objects.filter(
created__year=year, created__month=month, is_draft=False
).count() # <-- Add this line
# *** ADD note_count to month_count ***
month_count = entry_count + link_count + quote_count + photo_count + note_count # <-- Add note_count
if month_count:
counts = [
("entry", entry_count),
("link", link_count),
("photo", photo_count),
("quote", quote_count),
("note", note_count), # <-- Add this line
]
# ... (rest of the if block)
# *** ADD note_count to max_count ***
max_count = max(
max_count, entry_count, link_count, quote_count, photo_count, note_count # <-- Add note_count
)
# ... (rest of archive_year)
def archive_month(request, year, month):
# ... (initial setup)
# *** ADD Note to the list of models ***
for model, content_type in (
(Entry, "entry"),
(Quotation, "quotation"),
(Blogmark, "blogmark"),
(Note, "note"), # <-- Add this line
):
ids = model.objects.filter(
# ... (existing filters)
).values_list("id", flat=True)
items.extend(
# ... (existing extend logic)
)
# ... (rest of archive_month)
def archive_day(request, year, month, day):
# ... (initial setup)
# *** ADD Note to the list of models ***
for name, model in (
("blogmark", Blogmark),
("entry", Entry),
("quotation", Quotation),
("note", Note), # <-- Add this line
):
filt = model.objects.filter(
# ... (existing filters)
).order_by("created")
context[name] = list(filt)
count += len(context[name])
items.extend([{"type": name, "obj": obj} for obj in context[name]])
# ... (rest of archive_day)
# INTERSECTION_SQL definition remains the same
def archive_tag(request, tags, atom=False):
# ... (initial setup)
# *** ADD Note to the list of models ***
for model, content_type in (
(Entry, "entry"),
(Quotation, "quotation"),
(Blogmark, "blogmark"),
(Note, "note"), # <-- Add this line
):
cursor.execute(
INTERSECTION_SQL
% {
"content_table": "blog_%s" % content_type,
"tag_table": "blog_%s_tags" % content_type,
# *** Ensure the tag table key matches the content type ***
"tag_table_content_key": "%s_id" % content_type,
# ... (rest of the dict)
}
)
# ... (rest of the loop)
# ... (rest of archive_tag)
# ... (other views like about, series, etc.)
# *** ADD redirect_note HERE ***
def redirect_note(request, pk):
return Redirect(get_object_or_404(Note, pk=pk).get_absolute_url())
# ... (rest of file)
# *** Update api_add_tag ***
@require_POST
@staff_member_required
def api_add_tag(request):
# ... (existing code)
# *** ADD 'note': Note HERE ***
model = {"entry": Entry, "blogmark": Blogmark, "quotation": Quotation, "note": Note}.get(
content_type
)
# ... (rest of api_add_tag)6. Update URL Configuration:
- File:
config/urls.py - Action: Add a URL pattern for the
redirect_noteview.
# config/urls.py
# ... (other imports)
from blog import views as blog_views
urlpatterns = [
# ... (existing patterns)
re_path(r"^e/(\d+)/?$", blog_views.redirect_entry),
re_path(r"^b/(\d+)/?$", blog_views.redirect_blogmark),
re_path(r"^q/(\d+)/?$", blog_views.redirect_quotation),
# *** ADD redirect_note HERE ***
re_path(r"^n/(\d+)/?$", blog_views.redirect_note), # <-- Add this line
# ... (rest of patterns)
]
# ... (rest of file)7. Create the Template for Notes:
- File:
templates/note.html(Create this new file) - Action: Add the template content. You can copy the structure from
quotation.htmlorblogmark.htmland adapt it.
{# templates/note.html #}
{% extends "base.html" %}
{% load entry_tags %}
{% block title %}Note{% if note.is_draft %} [DRAFT]{% endif %}{% endblock %}
{% block extra_head %}
{% include "includes/metatags.html" with item=note content_type="note" %}
{% if note.is_draft %}
<meta name="robots" content="noindex">
{% endif %}
{% endblock %}
{% block body_class %}note{% endblock %}
{% block content %}
<div class="e note">
{% if note.is_draft %}
<p class="draft-warning">This is a draft post and is not publicly visible.</p>
{% endif %}
{{ note.body_rendered|xhtml2html|typography|safe }}
{% entry_footer note %}
{% include "includes/comments.html" %}
</div>
{% endblock %}
{% block nav_extra %}
{% include "includes/related_tags.html" with item=note %}
{% endblock %}8. Update the Mixed List Item Template:
- File:
templates/includes/blog_mixed_list_item.html(Assuming this file exists based onblog_tags.py) - Action: Add an
elifcondition to handle the 'note' type.
{# templates/includes/blog_mixed_list_item.html #}
{% load entry_tags %}
{% load humanize %}
<div class="{% if item.type == 'entry' %}e{% elif item.type == 'blogmark'%}l{% elif item.type == 'quotation'%}q{% elif item.type == 'note' %}n{% endif %}"> {# <-- Add 'n' class for notes #}
{# ... (Date logic if showdate is true) ... #}
{% if item.type == 'entry' %}
{# ... entry rendering ... #}
{% elif item.type == 'blogmark' %}
{# ... blogmark rendering ... #}
{% elif item.type == 'quotation' %}
{# ... quotation rendering ... #}
{# *** ADD Note rendering HERE *** #}
{% elif item.type == 'note' %}
<div class="body">
{{ item.obj.body_rendered|xhtml2html|typography|safe }}
</div>
<p class="meta">
{% if showdate %}{{ item.obj.created|naturalday }}{% endif %}
<a href="{{ item.obj.get_absolute_url }}">#</a>
{% for tag in item.obj.tags.all %}
<a href="{{ tag.get_absolute_url }}" rel="tag">{{ tag }}</a>
{% endfor %}
{% if user.is_staff %}
<a href="{{ item.obj.edit_url }}">edit</a>
{% endif %}
</p>
{% endif %}
</div>9. Update Feeds:
- File:
blog/feeds.py - Action: Modify
Everythingfeed class andsitemapfunction.
# blog/feeds.py
# ... (import Note at the top)
from blog.models import Entry, Blogmark, Quotation, Note # <-- Add Note
class Everything(Base):
# ... (existing attributes)
def items(self):
# ... (get last_30_entries, last_30_blogmarks, last_30_quotations)
# *** ADD last_30_notes HERE ***
last_30_notes = list(
Note.objects.filter(is_draft=False)
.prefetch_related("tags")
.order_by("-created")[:30]
)
# *** ADD last_30_notes to combined list ***
combined = last_30_blogmarks + last_30_entries + last_30_quotations + last_30_notes # <-- Add notes
combined.sort(key=lambda e: e.created, reverse=True)
return combined[:30]
def item_title(self, item):
if isinstance(item, Entry):
return item.title
elif isinstance(item, Blogmark):
return item.link_title
elif isinstance(item, Quotation):
return "Quoting %s" % item.source
# *** ADD Note title HERE ***
elif isinstance(item, Note):
# Simple title for notes in feeds
return "Note"
else:
# Fallback, should not happen
return "Item"
# ... (rest of feeds.py, update the description template feeds/everything.html as needed)
def sitemap(request):
# ... (existing xml start)
# *** ADD Note to the list of classes ***
for klass in (Entry, Blogmark, Quotation, Note): # <-- Add Note
for obj in klass.objects.only("slug", "created"):
# ... (existing xml append logic)
# ... (rest of sitemap)- File:
templates/feeds/everything.html(Create or modify) - Action: Add handling for
item.is_note.
{# templates/feeds/everything.html #}
{% load entry_tags %}
{% if item.is_entry %}
{# ... entry body ... #}
{% elif item.is_blogmark %}
{# ... blogmark body ... #}
{% elif item.is_quotation %}
{# ... quotation body ... #}
{% elif item.is_note %}
{# *** ADD Note body HERE *** #}
{{ item.body_rendered|xhtml2html|typography|safe }}
{% endif %}
{# ... (Optional: Link/source information if needed) ... #}10. Update Search:
- File:
blog/search.py - Action: Add
Noteto the list of searchable models.
# blog/search.py
# ... (import Note at the top)
from blog.models import Entry, Blogmark, Quotation, Tag, load_mixed_objects, Note # <-- Add Note
# ...
def search(request, q=None, return_context=False):
# ... (existing setup)
# *** ADD Note to the loop ***
for klass, type_name in (
(Entry, "entry"),
(Blogmark, "blogmark"),
(Quotation, "quotation"),
(Note, "note"), # <-- Add this line
):
if selected_type and selected_type != type_name:
continue
klass_qs = make_queryset(klass, type_name)
type_count = klass_qs.count()
if type_count:
type_counts_raw[type_name] = type_count
# *** Ensure tag counts work correctly with Note ***
for tag, count in (
Tag.objects.filter(**{"%s__in" % type_name: klass_qs}) # This dynamic lookup should work
.annotate(n=models.Count("tag"))
.values_list("tag", "n")
):
tag_counts_raw[tag] = tag_counts_raw.get(tag, 0) + count
# ... (year/month counts)
qs = qs.union(klass_qs.values(*values))
# ... (rest of search view)
# *** Update noun logic if needed ***
noun = {
"quotation": "Quotations",
"blogmark": "Blogmarks",
"entry": "Entries",
"note": "Notes", # <-- Add this line
}.get(selected.get("type")) or "Posts"
# ... (rest of search view)11. Update Context Processors:
- File:
blog/context_processors.py - Action: Add
Notetoyears_with_content.
# blog/context_processors.py
# ... (import Note at the top)
from blog.models import Entry, Blogmark, Quotation, Note # <-- Add Note
# ...
def years_with_content():
# *** CHANGE cache_key to invalidate old cache ***
cache_key = "years-with-content-3" # <-- Increment key
years = cache.get(cache_key)
if not years:
years = list(
set(
list(Entry.objects.datetimes("created", "year"))
+ list(Blogmark.objects.datetimes("created", "year"))
+ list(Quotation.objects.datetimes("created", "year"))
# *** ADD Note datetimes HERE ***
+ list(Note.objects.datetimes("created", "year")) # <-- Add this line
)
)
years.sort()
cache.set(cache_key, years, 60 * 60)
return years12. Update Tag Cloud Templetetag:
- File:
blog/templatetags/tag_cloud.py - Action: Add query for Note tags.
# blog/templatetags/tag_cloud.py
# ... (imports)
@register.inclusion_tag("includes/tag_cloud.html")
def tag_cloud():
# ... (existing connection and cursor setup)
cursor.execute(
"select tag from blog_entry_tags, blog_tag where blog_entry_tags.tag_id = blog_tag.id"
)
entry_tags = [row[0] for row in cursor.fetchall()]
cursor.execute(
"select tag from blog_blogmark_tags, blog_tag where blog_blogmark_tags.tag_id = blog_tag.id"
)
blogmark_tags = [row[0] for row in cursor.fetchall()]
cursor.execute(
"select tag from blog_quotation_tags, blog_tag where blog_quotation_tags.tag_id = blog_tag.id"
)
quotation_tags = [row[0] for row in cursor.fetchall()]
# *** ADD Note tags query HERE ***
cursor.execute(
"select tag from blog_note_tags, blog_tag where blog_note_tags.tag_id = blog_tag.id"
)
note_tags = [row[0] for row in cursor.fetchall()] # <-- Add this
cursor.close()
# Add them together
# *** ADD note_tags to the list ***
tags = entry_tags + blogmark_tags + quotation_tags + note_tags # <-- Add note_tags
return _tag_cloud_helper(tags)13. Update Tags Autocomplete View:
- File:
blog/tag_views.py - Action: Add note counts to the annotation.
# blog/tag_views.py
# ... (imports)
def tags_autocomplete(request):
# ... (existing query setup)
# *** Subquery for counting notes ***
note_count = (
Tag.objects.filter(id=OuterRef("pk"))
.annotate(
count=Count("note", filter=Q(note__is_draft=False), distinct=True) # <-- Use 'note' model name
)
.values("count")
)
tags = (
Tag.objects.filter(tag__icontains=query)
.annotate(
total_entry=Subquery(entry_count),
total_blogmark=Subquery(blogmark_count),
total_quotation=Subquery(quotation_count),
# *** Annotate with note count ***
total_note=Subquery(note_count), # <-- Add this line
is_exact_match=Case(
When(tag__iexact=query, then=Value(1)),
default=Value(0),
output_field=IntegerField(),
),
)
.annotate(
# *** Add note count to total count ***
count=F("total_entry") + F("total_blogmark") + F("total_quotation") + F("total_note") # <-- Add F("total_note")
)
.order_by("-is_exact_match", "-count", Length("tag"))[:5]
)
# ... (rest of the view)
# *** Update the debug output values if needed (or rely on .values()) ***
# The .values() call will automatically include the new annotations.
# Example updated manual dictionary for debug output (if you were building it manually):
# return JsonResponse({
# "tags": [
# {
# "id": t.id, "tag": t.tag, "description": t.description,
# "total_entry": t.total_entry, "total_blogmark": t.total_blogmark,
# "total_quotation": t.total_quotation, "total_note": t.total_note, # <-- Add total_note
# "is_exact_match": t.is_exact_match, "count": t.count
# } for t in tags
# ]
# })
# But since it uses .values(), no change is needed for the main return.
return JsonResponse({"tags": list(tags.values())})Summary of Changes:
- Model: Defined
NoteinheritingBaseModel. - Admin: Registered
Notefor admin interface. - Migrations: Created and applied schema changes.
- Views: Updated main content views (
archive_item,index, archives, tags) to includeNote. Added redirect view. - URLs: Added URL pattern for the redirect view.
- Templates: Created
note.htmland updated the shared list item templateblog_mixed_list_item.html. - Feeds: Added
Noteto theEverythingfeed andsitemap. Updated feed item template. - Search: Included
Notein search queries and results processing. - Context Processor: Added
Notetoyears_with_content. - Templatetags: Updated
tag_cloudandtags_autocompleteto count notes.
After adding this code and running the migrations, your new 'Note' content type should be fully integrated. Remember to potentially add tests in blog/tests.py and a NoteFactory in blog/factories.py for better robustness.
50,657 input, 7,995 output, {"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 50657}]}