lemon24

"""
I used https://pythonspeed.com/products/filmemoryprofiler/
on https://github.com/lemon24/reader

Saw that feedparser accounted for a big part of the memory used.

The monkeypatch below avoids some encode()/decode() calls in the 
encoding detection heuristics used in feedparser.encodings.convert_to_utf8();
https://github.com/kurtmckee/feedparser/blob/6.0.2/feedparser/encodings.py#L71

lemon24

from datetime import datetime, timezone, timedelta
from flask.json.tag import TaggedJSONSerializer    
from itsdangerous import Serializer, URLSafeSerializer
import json 


class DatetimeTag:
    
    key = '__d'
    

lemon24

"""
None-like singleton in Python (mypy-compatible).

https://github.com/lemon24/reader/issues/177#issuecomment-674786498

"""

from typing import Union, cast

lemon24

"""
Extract query timings from (stderr) log output of:

    reader --debug-storage [update | update --new-only | search update] --vv

Note that the timings are not entirely accurate:

* for SELECTs, a read lock may be still held after cursor.execute() returns
* we don't know how long transactions take; BEGIN IMMEDIATE tells us how long
  it waited for a write lock; BEGIN (DEFERRED) doesn't tell us anything,

lemon24

import os
import os.path
import time
import sys
import random
import itertools
from datetime import datetime
from subprocess import run
from threading import Thread
import threading

lemon24

"""
Run `reader update` and `reader update --new-only && reader search update``
in parallel, like cron would, but at a "scaled down" timeframe.

We use vcrpy to avoid making network calls.

https://github.com/lemon24/reader/issues/175

"""
import os

lemon24

This gist tracks the creation of an almost fully functional sqlite3 server in Python 3, using only the multiprocessing.managers standard library module.

But why?

• To see if it can be done.
• To learn about multiprocessing managers.
• Aside from whole-database locking, SQLite (maybe, see below) supports table-level locking between multiple connections in the same process to the same database. A "SQLite server" makes it possible for users in different processes to use SQLite connections that live in the same process.

---

lemon24

import collections
import textwrap
import functools


class Query(collections.OrderedDict):

    indent_prefix = '    '
    default_separators = dict(WHERE='AND', HAVING='AND')

lemon24

import shutil

import mutagen

shutil.copy('original.mp4', 'new.mp4')

# mutagen.File knows how to open any file (works with both MP4 and MP4):
#
# https://mutagen.readthedocs.io/en/latest/user/gettingstarted.html    
# https://mutagen.readthedocs.io/en/latest/api/base.html#mutagen.File

lemon24

>>> import math, collections                                                                          
>>> from reader import *                                                                              
>>>                                                                                                   
>>> reader = make_reader('db.sqlite')                                                                 
>>>                                                                                                   
>>> def word_count_magnitude(text): 
...     if not text: 
...         return 0 
...     return int(math.log10(len(text.split())))  
...