pycon LT talk demo, see https://slides.com/samuelcolvin/garbage-in-pydantic-youre-golden

pycon_lt_analyse_data.py

import polars as pl

pl.Config.set_tbl_rows(30)

ds = pl.read_parquet('page_views.parquet').sort('ts')

path_counts = (
    ds.with_columns(pl.col('path').str.replace(r'^/(latest|dev-v\d|\d\.\d+)', ''))
    .groupby(pl.col('path'))
    .agg([pl.count().alias('count')])
    .with_columns((pl.col('count') / pl.sum('count') * 100).alias('percentage'))
    .sort('count', descending=True)
    .head(30)
)
print(path_counts)

pycon_lt_read_data.py

from datetime import datetime
from pathlib import Path
import polars as pl

from pydantic import BaseModel, Field, TypeAdapter, AliasPath
from devtools import debug


class PageView(BaseModel):
    user_id: str
    ts: datetime = Field(validation_alias=AliasPath('ts', '$date'))
    path: str = Field('?', validation_alias=AliasPath('url', 'path'))
    referer: str = Field('?')


data = Path('docs_pydantic_dev.json').read_bytes()
ta = TypeAdapter(list[PageView])

with debug.timer('load data into Pydantic'):
    # takes 57s for just `json.loads`
    models = ta.validate_json(data)

print(f'loaded {len(models)} models')
debug(models[:3])

with debug.timer('create df'):
    # load the data directly into
    df = pl.DataFrame(models)
print(df)
df.write_parquet('page_views.parquet')