wshayes

#!/usr/bin/env python
# -*- coding: utf-8-*-

"""
Webscrape Intercom Help Articles and export them into markdown and html as a JSON data file

This captures Collection info but not Sections.  Images are downloaded into an images folder.
The images are renamed with the collection directory name (see the dir_map dict below) in numerical order.
You will need to dedupe duplicate images using another tool. Renaming the images avoids bad initial names
and duplicate image filenames (quick hack and not ideal - feel free to improve as desired.)

wshayes

# syntax=docker/dockerfile:experimental

FROM python:3.7-slim AS base

# ENV LANG=C.UTF-8  # Sets utf-8 encoding for Python et al
# ENV PYTHONDONTWRITEBYTECODE=1  # Turns off writing .pyc files; superfluous on an ephemeral container.
# ENV PYTHONUNBUFFERED=1  # Seems to speed things up

ENV PYTHONUNBUFFERED=1 \
    PYTHONDONTWRITEBYTECODE=1 \

wshayes

token: <Github Personal Access token with github actions/secrets scopes>
username: <github username>
globals:
  - name: <secret_name>
    value: <value>
    description: <description>

wshayes

traefik:
    image: traefik:1.7
    container_name: traefik
    ports:
      - 80:80
      - 443:443
    command:
      - --api
      - --debug=false
      - --logLevel=ERROR

wshayes

# https://blog.ionelmc.ro/2020/01/20/is-there-anything-safe-in-python/

def safe_repr(obj, maxdepth=5):
    if not maxdepth:
        return '...'
    obj_type = type(obj)
    obj_type_type = type(obj_type)
    newdepth = maxdepth - 1

    # only represent exact builtins

wshayes

# https://github.com/tiangolo/fastapi/issues/883#issuecomment-575913215
# You can probably use an async queue so your MQTT client will push messages to the queue and the WS server will get from the queue and send them to the WS client.

    from asyncio import Queue

    queue: Queue = None

    @app.websocket("/ws")
    async def websocket_endpoint(websocket: WebSocket):
        await websocket.accept()

wshayes

# https://stackoverflow.com/a/13530258/886938

import multiprocessing as mp
import time

fn = 'c:/temp/temp.txt'

def worker(arg, q):
    '''stupidly simulates long running process'''
    start = time.clock()

wshayes

Tips and tricks

Sources

• https://martinheinz.dev/blog/1

Skipping Begining of Iterable

Sometimes you have to work with files which you know that start with variable number of unwanted lines such as comments. itertools again provides easy solution to that:

wshayes

# https://gitter.im/tiangolo/fastapi?at=5db608eeef84ab3786aba5a4
from fastapi import FastAPI, Depends
from starlette.testclient import TestClient

app = FastAPI()


def capture_exception(exc: Exception) -> None:
    print(str(exc))

wshayes

# https://github.com/tiangolo/fastapi/issues/413#issuecomment-517504748

# @Rehket for what it's worth, I recommend subclassing BaseModel and using that as the root for all of your API models (to reduce code repetition). For example, I use a base class that mostly looks like this:

import re
from functools import partial
from typing import Any, Dict

from fastapi.encoders import jsonable_encoder
from pydantic import BaseConfig, BaseModel