codimd-index create an index for codimd

README.md

codimd-index: maintain a global index for codimd pages.

We have a team codimd and it is wonderful. We have CMD_ALLOW_FREEURL=true so that users can create alias'd (named) pages.

Users can't see each others pages though, unless they know to look for them (codimd/#975).

This is just a "manually updated" script to generate a simple index page. It'd be wonderful if codimd/#103 was fixed, then we could generate a table and let user sort.

To run it, you need pyscopg2 (apt install python3-psycopg2).

Then:

• create a page with the alias index (--name). (Just go to /name if you have CMD_ALLOW_FREEURL on).

• run codi-index:

   ./codi-index.py --host=your.dbhost --password=your.db.password
  

Notes:

• The index only edits named or aliased. That would be easy enough to change, but this way temporary docs didn't show up. We use CMD_ALLOW_FREEURL=true to allow the creation of named pages.
• I'm not sure I have the inserts to the Revision table correct, when you look at the revisions in the web-ui, it does not show content for the current revision.

b64-to-uuid

#!/usr/bin/python3
"""
Convert a uuid to base64 string or base64 string to uuid.

   $ b64-to-uuid ApcZRUYIQM25W0CRCudF8w
   uuid   ApcZRUYIQM25W0CRCudF8w -> base64 02971945460840cdb95b40910ae745f3

   $ b64-to-uuid 02971945-4608-40cd-b95b-40910ae745f3
   base64 02971945-4608-40cd-b95b-40910ae745f3 -> uuid ApcZRUYIQM25W0CRCudF8w

"""

import base64
import sys
import uuid
import binascii

str_in = sys.argv[1]

def uuid2b64(u):
    b = binascii.unhexlify(u.replace("-", ""))
    return base64.urlsafe_b64encode(b).decode('utf-8').rstrip("=")

def b642uuid(b):
    return str(uuid.UUID(bytes=base64.urlsafe_b64decode(b + '=' * (4 - len(b) % 4))))

try:
    uuid.UUID(str_in)
    in_type = "uuid"
except ValueError:
    in_type = "base64"

if in_type == "uuid":
    print("uuid %s -> base64 %s" % (str_in, uuid2b64(str_in)))
else:
    print("base64 %s -> uuid %s" % (str_in, b642uuid(str_in)))

bootstrap

#!/bin/sh
MY_REPO="https://gist.github.com/2ea3542c4cb2806b0f334c273a219cdc.git"
MY_NAME="codimd-index"
MY_D="/$MY_NAME"
INDEX_LOOP=2h
VERBOSITY=${VERBOSITY:-2}

msg() {
    # message(level, message)
    [ "$VERBOSITY" -ge "$1" ] || return
    shift
    stderror "$@"
}
info() { msg 1 "$@"; }
debug() { msg 2 "$@"; }
error() { msg 0 "$@"; }
stderror() { echo "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }

has_cmd() {
    command -v "$1" >/dev/null
}

get_deps() {
    local pkgs=""
    has_cmd git || pkgs="$pkgs git"
    if ! has_cmd python3; then
        pkgs="$pkgs python3-minimal python3-psycopg2"
    else
        python3 -c 'import psycopg2' >/dev/null 2>&1 ||
            pkgs="$pkgs python3-psycopg2"
    fi
    [ -n "$pkgs" ] || { debug "no packages to install"; return 0; }

    pkgs="${pkgs# }"
    debug "installing packages $pkgs"
    if [ -z "$_APT_UPDATED" ]; then
        apt-get update || fail "failed apt-get update"
        _APT_UPDATED="true"
    fi
    DEBIAN_FRONTEND=noninteractive \
        apt-get install --no-install-recommends --assume-yes $pkgs || {
        error "failed install $pkgs"
    }
}

loop_run() {
    local naplen=${INDEX_LOOP:-2h} r=""
    set -- ./codi-index "$@"
    while :; do
        info "$(date -R): running index: $*"
        "$@"
        r="$?"
        info "$(date -R): returned $r. sleeping $naplen"
        if [ $r -ne 0 ]; then
            if [ "${INDEX_LOOP_IGNORE_ERRORS:-false}" != "false" ]; then
                error "failed - $r"
                return $r
            fi
        fi
        sleep $naplen || fail "sleep failed ($?) bye now"
    done
}

if [ -z "$GIT_REPO" ]; then
    if [ -n "$BS_URL" ]; then
        case "$BS_URL" in
            https://gist.githubusercontent.com/*/bootstrap) :;;
            *) fail "can't get a git repo from BS_URL $BS_URL";;
        esac
        # just take up to 'raw'.
        base="https://gist.github.com/"
        user_hash=${BS_URL#https://*/}
        user_hash=${user_hash%%/raw/*}
        ghash=${user_hash#*/}
        GIT_REPO="${base}/$ghash.git"
        debug "using GIT_REPO=$GIT_REPO from BS_URL"
    else
        debug "using default GIT_REPO=$GIT_REPO"
        GIT_REPO="$MY_REPO"
    fi
fi

get_deps || fail

pdir=$(dirname "${MY_D}")
bdir=$(basename "${MY_D}")
[ -d "$pdir" ] || mkdir -p "$pdir" ||
    fail "failed to create parent dir $pdir for MY_D=$MY_D"

if [ ! -d "$MY_D" ]; then
    cd "$pdir" || fail "failed chdir to $pdir"
    debug "git clone $GIT_REPO $bdir"
    git clone "$GIT_REPO" "$bdir" ||
        fail "failed clone of $GIT_REPO in $pdir"
    cd "$bdir" || fail "failed cd $MY_D"
else
    cd "$MY_D" || fail "failed cd $MY_D"
    git fetch && git merge --ff-only || fail "failed to fetch and merge."
fi

if [ -n "$INDEX_LOOP" -a "$INDEX_LOOP" != "0" ]; then
    loop_run "$@"
else
    debug "not running index in loop: INDEX_LOOP=$INDEX_LOOP"
fi

codi-index

#!/usr/bin/env python3
"""
Make/maintain an index page for codimd that uses postgres.

   $ codi-index --user=codimd --host=database.codimd --password=foo

Updates the page with alias '--name' (default='index') with an index.
"""

import argparse
import datetime
import io
import logging
import os
import sys
import textwrap
import psycopg2
import uuid

# def uuid2b64(u):
#     # codi strips off trailing '='.
#     return base64.b64encode(uuid.UUID(u).bytes).decode('utf-8').rstrip("=")
#
#
# def b642uuid(b):
#     # base64 values should be length of multiple of 4.
#     # you can just add = at the end, and python's b64decode
#     # will strip off the correct number.
#     return str(uuid.UUID(bytes=base64.b64decode(b + "====")).hex)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--verbose', '-v', action='count', default=0)

    parser.add_argument(
        '-u', '--user', help='username to connect to postgres',
        default="codimd")
    parser.add_argument(
        '-P', '--password', help='password for user',
        default=os.environ.get('PGPASSWORD'))
    parser.add_argument(
        '-d', '--dbname', help='The name of the database',
        default="codimd")
    parser.add_argument(
        '-H', '--hostname', help='connect to db on host HOSTNAME',
        default="localhost"),
    parser.add_argument(
        '-n', '--name', help='The name of the document',
        default="index")
    parser.add_argument(
        '--output-only', help='output-only (do not udpate database)',
        action='store', default=None)

    args = parser.parse_args()

    level = (logging.ERROR, logging.INFO, logging.DEBUG)[
        min(args.verbose + 1, 2)]
    logging.basicConfig(level=level)

    conn = psycopg2.connect(
        host=args.hostname, database=args.dbname, user=args.user,
        password=args.password)

    cur = conn.cursor()

    cur.execute(
        """SELECT id, content from "Notes" WHERE "Notes".alias = %s;""",
        (args.name,))
    if cur.rowcount != 1:
        logging.error("Failed to get a page named %s" % args.name)
        sys.exit(1)

    index_uuid, existing_content = cur.fetchone()
    logging.debug("uuid of index page is %s", index_uuid)

    cur.execute(textwrap.dedent("""
        SELECT "Notes"."id","Notes"."alias","Notes"."title","Users"."email", "Users"."profileid"
           FROM "Notes" INNER JOIN "Users"
           ON "Notes"."ownerId" = "Users"."id"
           WHERE "Notes"."alias" != ''
           ORDER BY "Users"."email", "Users"."profileid", "Notes"."alias";
        """))

    pages = cur.rowcount
    logging.info("There are %d pages found. index page '%s' is uuid %s",
                 pages, args.name, index_uuid)

    if pages < 3:
        logging.error("only found %d pages, that seems odd", pages)
        sys.exit(1)

    title = "index"
    buf = io.StringIO()
    buf.write("# " + title + "\n")
    last_owner = ""

    for (cur_uuid, alias, title, email, profileid) in cur:
        if email:
            owner = email
        elif profileid.startswith("LDAP-"):
            owner = profileid[len("LDAP-"):]
        else:
            owner = profileid

        if owner != last_owner:
            if last_owner != "":
                buf.write("\n")
            last_owner = owner
            buf.write("## %s \n" % owner)

        buf.write(" * [%s](/%s): %s\n" % (alias, alias, title))

    content = buf.getvalue()

    if content == existing_content:
        logging.info("Page '%s' does not need update\n", args.name)
        sys.exit(0)

    if args.output_only:
        with open(args.output_only, "w") as fp:
            fp.write(content)
        logging.info("Wrote to %s\n", args.output_only)
        sys.exit(0)

    now = datetime.datetime.now()
    cur.execute(
        textwrap.dedent("""\
        UPDATE "Notes" set
           "content" = %s,
           "lastchangeAt" = %s,
           "updatedAt" = %s,
           "title" = %s
        WHERE id = %s;
        """),
        (content, now, now, title, index_uuid))

    if cur.rowcount != 1:
        logging.error(
            "Updated updated %d rows, expected 1: %s",
            cur.rowcount, cur.statusmessage)
        sys.exit(1)

    logging.debug("update returned %s.", cur.statusmessage)

    revId = uuid.uuid4()
    cur.execute(
        textwrap.dedent("""\
        INSERT into "Revisions"
            ("id", "noteId", "content", "length", "createdAt", "updatedAt")
        VALUES (%s, %s,%s,%s,%s,%s);
        """),
        (str(revId), index_uuid, content, len(content), now, now))

    if cur.rowcount != 1:
        logging.error(
            "Failed insert into Revisions (%d rows added): %s",
            cur.rowcount, cur.statusmessage)
    logging.debug("added revision %s for noteId %s", revId, index_uuid)

    conn.commit()

    logging.info("Updated page %s NoteId=%s RevisionId=%s\n",
                 args.name, index_uuid, revId)

    cur.close()

    sys.exit(0)


if __name__ == "__main__":
    main()

codi-set-owner

#!/usr/bin/env python3
"""
Set author of a codi note.

$ codi-set-author [--note=|--owner=] new-user

    new-user can be uuid or email
"""

import argparse
import base64
import datetime
import io
import logging
import os
import sys
import textwrap
import psycopg2
import uuid

def is_uuid(u):
    try:
        uuid.UUID(u)
    except ValueError:
        return False

# def uuid2b64(u):
#     # codi strips off trailing '='.
#     return base64.urlsafe_b64encode(uuid.UUID(u).bytes).decode('utf-8').rstrip("=")
#
#

def b642uuid(b):
     # base64 values should be length of multiple of 4.
     # you can just add = at the end, and python's b64decode
     # will strip off the correct number.
    return str(uuid.UUID(bytes=base64.urlsafe_b64decode(b + '=' * (4 - len(b) % 4))))

def get_user_by_input(info, cursor):
    if is_uuid(info):
        cursor.execute(textwrap.dedent("""
            SELECT "Users".id, "Users".email, "Users".profileid
            FROM "Users" WHERE "Users".id = %s"""),
                    (uuid.UUID(info),))
    else:
        cursor.execute(textwrap.dedent("""
            SELECT "Users".id, "Users".email, "Users".profileid
            FROM "Users" 
            WHERE ("Users".email = %s OR "Users".profileid = %s)"""),
            (info, info))

    if cursor.rowcount != 1:
        logging.error("Found %d matches for user '%s':", cursor.rowcount, info)
        for (cuuid, email, profileid) in cursor:
            logging.error("uuid=%s email=%s profileid=%s", cuuid, email, profileid)
        sys.exit(1)
    return cursor.fetchone()
    

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--verbose', '-v', action='count', default=0)
    parser.add_argument('--dry-run', '-n', action='store_true')

    parser.add_argument(
        '-u', '--user', help='username to connect to postgres',
        default="codimd")
    parser.add_argument(
        '-P', '--password', help='password for user',
        default=os.environ.get('PGPASSWORD'))
    parser.add_argument(
        '-d', '--dbname', help='The name of the database',
        default="codimd")
    parser.add_argument(
        '-H', '--hostname', help='connect to db on host HOSTNAME',
        default="localhost"),

    from_group = parser.add_mutually_exclusive_group(required=True)
    from_group.add_argument(
        '--owner', default=False, action='store_true',
        help='owner as email or uuid')
    from_group.add_argument(
        '--page-uuids', default=False, action='store_true',
        help='from-list are page uuids (or base64 encoded uuids)')
    from_group.add_argument(
        '--page-names', default=False, action='store_true',
        help='from-list are page names (aliases)')

    parser.add_argument('new_owner', help='new owner as email or uuid')
    parser.add_argument('from_list', help='pages/notes or owner',
                        nargs="+")

    args = parser.parse_args()

    level = (logging.ERROR, logging.INFO, logging.DEBUG)[
        min(args.verbose + 1, 2)]
    logging.basicConfig(level=level)

    conn = psycopg2.connect(
        host=args.hostname, database=args.dbname, user=args.user,
        password=args.password)

    cur = conn.cursor()

    (new_uuid, new_email, new_profileid) = get_user_by_input(args.new_owner, cur)
    
    if args.owner:
        # from-list is an owner by uuid or email.
        if len(args.from_list) > 1:
            logging.error("only a single owner supported (%s)", args.from_list)
            sys.exit(1)
            
        (owner_uuid, owner_email, owner_profileid) = get_user_by_input(args.from_list[0], cur)

        cur.execute(textwrap.dedent("""
            SELECT "Users".id, "Notes".id 
            FROM "Users" INNER JOIN "Notes"
            ON "Notes"."ownerId" = "Users"."id"
            WHERE "Users".id = %s"""),
            (owner_uuid,))

        page_uuids = []
        for (cur_uuid, page_uuid) in cur:
            page_uuids.append(page_uuid)
    
        logging.info("Found %d pages owned by %s (uuid=%s)",
                     len(page_uuids), args.from_list[0], owner_uuid)

    elif args.page_uuids:
        # pages are base64 uuids or uuids
        page_uuids = []
        for page in args.from_list:
            if is_uuid(page):
                page_uuids.append(page)
            else:
                try:
                    page_uuids.append(b642uuid(page.rstrip("#")))
                except ValueError:
                    logging.error("%s is not a uuid or base64 uuid", page)
                    sys.exit(1)

        uuids = tuple(str(uuid.UUID(p)) for p in page_uuids)
        cur.execute(textwrap.dedent("""
            SELECT "Notes".id 
            FROM "Notes"
            WHERE "Notes".id in %s"""),
            (uuids,))

        if cur.rowcount != len(page_uuids):
            logging.error("you gave %d input ids, but %d exist as notes",
                          len(page_uuids), cur.rowcount)
            sys.exit(1)

    else:
        # --page-names (aliases)
        cur.execute(textwrap.dedent("""
            SELECT "Notes".id, "Notes".alias
            FROM "Notes"
            WHERE "Notes".alias in %s"""),
            (tuple(args.from_list),))

        if cur.rowcount != len(args.from_list):
            logging.error("you gave %d pages but %d exist",
                          len(args.from_list), cur.rowcount)
            logging.error("Existing were:")
            for (cuuid, alias) in cur:
                logging.error("%s %s", cuuid, alias)
            sys.exit(1)

        page_uuids = [x[0] for x in cur]

    if len(page_uuids) == 0:
        logging.info("Nothing to do, 0 relevant page ids")
        sys.exit(0)

    owner_friendly = new_profileid if new_profileid else new_email
    logging.info("Will change ownership of %d pages to %s (%s)",
                 len(page_uuids), owner_friendly, new_uuid)
                 
    cur.execute(textwrap.dedent("""
        UPDATE "Notes" set "ownerId" = %s
        WHERE id in %s
    """), (new_uuid, tuple(page_uuids),))

    if cur.rowcount != len(page_uuids):
        logging.error("Updated %d pages, but expected to update %d",
                      cur.rowcount, len(page_uuids))
        cur.rollback()
        sys.exit(1)

    logging.info("Updated %d pages, setting owner to %s (%s)",
                 cur.rowcount, owner_friendly, new_uuid)

    if args.dry_run:
        logging.info("Dry run, rolling back\n")
        conn.rollback()
    else:
        conn.commit()

    conn.close()

    sys.exit(0)


if __name__ == "__main__":
    main()