Skip to content

Instantly share code, notes, and snippets.

@smoser

smoser/README.md

Last active Jul 22, 2020
Embed
What would you like to do?
codimd-index create an index for codimd

codimd-index: maintain a global index for codimd pages.

We have a team codimd and it is wonderful. We have CMD_ALLOW_FREEURL=true so that users can create alias'd (named) pages.

Users can't see each others pages though, unless they know to look for them (codimd/#975).

This is just a "manually updated" script to generate a simple index page. It'd be wonderful if codimd/#103 was fixed, then we could generate a table and let user sort.

To run it, you need pyscopg2 (apt install python3-psycopg2).

Then:

  • create a page with the alias index (--name). (Just go to /name if you have CMD_ALLOW_FREEURL on).

  • run codi-index:

     ./codi-index.py --host=your.dbhost --password=your.db.password
    

Notes:

  • The index only edits named or aliased. That would be easy enough to change, but this way temporary docs didn't show up. We use CMD_ALLOW_FREEURL=true to allow the creation of named pages.
  • I'm not sure I have the inserts to the Revision table correct, when you look at the revisions in the web-ui, it does not show content for the current revision.
#!/usr/bin/python3
"""
Convert a uuid to base64 string or base64 string to uuid.
$ b64-to-uuid ApcZRUYIQM25W0CRCudF8w
uuid ApcZRUYIQM25W0CRCudF8w -> base64 02971945460840cdb95b40910ae745f3
$ b64-to-uuid 02971945-4608-40cd-b95b-40910ae745f3
base64 02971945-4608-40cd-b95b-40910ae745f3 -> uuid ApcZRUYIQM25W0CRCudF8w
"""
import base64
import sys
import uuid
import binascii
str_in = sys.argv[1]
def uuid2b64(u):
b = binascii.unhexlify(u.replace("-", ""))
return base64.urlsafe_b64encode(b).decode('utf-8').rstrip("=")
def b642uuid(b):
return str(uuid.UUID(bytes=base64.urlsafe_b64decode(b + '=' * (4 - len(b) % 4))))
try:
uuid.UUID(str_in)
in_type = "uuid"
except ValueError:
in_type = "base64"
if in_type == "uuid":
print("uuid %s -> base64 %s" % (str_in, uuid2b64(str_in)))
else:
print("base64 %s -> uuid %s" % (str_in, b642uuid(str_in)))
#!/bin/sh
MY_REPO="https://gist.github.com/2ea3542c4cb2806b0f334c273a219cdc.git"
MY_NAME="codimd-index"
MY_D="/$MY_NAME"
INDEX_LOOP=2h
VERBOSITY=${VERBOSITY:-2}
msg() {
# message(level, message)
[ "$VERBOSITY" -ge "$1" ] || return
shift
stderror "$@"
}
info() { msg 1 "$@"; }
debug() { msg 2 "$@"; }
error() { msg 0 "$@"; }
stderror() { echo "$@" 1>&2; }
fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
has_cmd() {
command -v "$1" >/dev/null
}
get_deps() {
local pkgs=""
has_cmd git || pkgs="$pkgs git"
if ! has_cmd python3; then
pkgs="$pkgs python3-minimal python3-psycopg2"
else
python3 -c 'import psycopg2' >/dev/null 2>&1 ||
pkgs="$pkgs python3-psycopg2"
fi
[ -n "$pkgs" ] || { debug "no packages to install"; return 0; }
pkgs="${pkgs# }"
debug "installing packages $pkgs"
if [ -z "$_APT_UPDATED" ]; then
apt-get update || fail "failed apt-get update"
_APT_UPDATED="true"
fi
DEBIAN_FRONTEND=noninteractive \
apt-get install --no-install-recommends --assume-yes $pkgs || {
error "failed install $pkgs"
}
}
loop_run() {
local naplen=${INDEX_LOOP:-2h} r=""
set -- ./codi-index "$@"
while :; do
info "$(date -R): running index: $*"
"$@"
r="$?"
info "$(date -R): returned $r. sleeping $naplen"
if [ $r -ne 0 ]; then
if [ "${INDEX_LOOP_IGNORE_ERRORS:-false}" != "false" ]; then
error "failed - $r"
return $r
fi
fi
sleep $naplen || fail "sleep failed ($?) bye now"
done
}
if [ -z "$GIT_REPO" ]; then
if [ -n "$BS_URL" ]; then
case "$BS_URL" in
https://gist.githubusercontent.com/*/bootstrap) :;;
*) fail "can't get a git repo from BS_URL $BS_URL";;
esac
# just take up to 'raw'.
base="https://gist.github.com/"
user_hash=${BS_URL#https://*/}
user_hash=${user_hash%%/raw/*}
ghash=${user_hash#*/}
GIT_REPO="${base}/$ghash.git"
debug "using GIT_REPO=$GIT_REPO from BS_URL"
else
debug "using default GIT_REPO=$GIT_REPO"
GIT_REPO="$MY_REPO"
fi
fi
get_deps || fail
pdir=$(dirname "${MY_D}")
bdir=$(basename "${MY_D}")
[ -d "$pdir" ] || mkdir -p "$pdir" ||
fail "failed to create parent dir $pdir for MY_D=$MY_D"
if [ ! -d "$MY_D" ]; then
cd "$pdir" || fail "failed chdir to $pdir"
debug "git clone $GIT_REPO $bdir"
git clone "$GIT_REPO" "$bdir" ||
fail "failed clone of $GIT_REPO in $pdir"
cd "$bdir" || fail "failed cd $MY_D"
else
cd "$MY_D" || fail "failed cd $MY_D"
git fetch && git merge --ff-only || fail "failed to fetch and merge."
fi
if [ -n "$INDEX_LOOP" -a "$INDEX_LOOP" != "0" ]; then
loop_run "$@"
else
debug "not running index in loop: INDEX_LOOP=$INDEX_LOOP"
fi
#!/usr/bin/env python3
"""
Make/maintain an index page for codimd that uses postgres.
$ codi-index --user=codimd --host=database.codimd --password=foo
Updates the page with alias '--name' (default='index') with an index.
"""
import argparse
import datetime
import io
import logging
import os
import sys
import textwrap
import psycopg2
import uuid
# def uuid2b64(u):
# # codi strips off trailing '='.
# return base64.b64encode(uuid.UUID(u).bytes).decode('utf-8').rstrip("=")
#
#
# def b642uuid(b):
# # base64 values should be length of multiple of 4.
# # you can just add = at the end, and python's b64decode
# # will strip off the correct number.
# return str(uuid.UUID(bytes=base64.b64decode(b + "====")).hex)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument(
'-u', '--user', help='username to connect to postgres',
default="codimd")
parser.add_argument(
'-P', '--password', help='password for user',
default=os.environ.get('PGPASSWORD'))
parser.add_argument(
'-d', '--dbname', help='The name of the database',
default="codimd")
parser.add_argument(
'-H', '--hostname', help='connect to db on host HOSTNAME',
default="localhost"),
parser.add_argument(
'-n', '--name', help='The name of the document',
default="index")
parser.add_argument(
'--output-only', help='output-only (do not udpate database)',
action='store', default=None)
args = parser.parse_args()
level = (logging.ERROR, logging.INFO, logging.DEBUG)[
min(args.verbose + 1, 2)]
logging.basicConfig(level=level)
conn = psycopg2.connect(
host=args.hostname, database=args.dbname, user=args.user,
password=args.password)
cur = conn.cursor()
cur.execute(
"""SELECT id, content from "Notes" WHERE "Notes".alias = %s;""",
(args.name,))
if cur.rowcount != 1:
logging.error("Failed to get a page named %s" % args.name)
sys.exit(1)
index_uuid, existing_content = cur.fetchone()
logging.debug("uuid of index page is %s", index_uuid)
cur.execute(textwrap.dedent("""
SELECT "Notes"."id","Notes"."alias","Notes"."title","Users"."email", "Users"."profileid"
FROM "Notes" INNER JOIN "Users"
ON "Notes"."ownerId" = "Users"."id"
WHERE "Notes"."alias" != ''
ORDER BY "Users"."email", "Users"."profileid", "Notes"."alias";
"""))
pages = cur.rowcount
logging.info("There are %d pages found. index page '%s' is uuid %s",
pages, args.name, index_uuid)
if pages < 3:
logging.error("only found %d pages, that seems odd", pages)
sys.exit(1)
title = "index"
buf = io.StringIO()
buf.write("# " + title + "\n")
last_owner = ""
for (cur_uuid, alias, title, email, profileid) in cur:
if email:
owner = email
elif profileid.startswith("LDAP-"):
owner = profileid[len("LDAP-"):]
else:
owner = profileid
if owner != last_owner:
if last_owner != "":
buf.write("\n")
last_owner = owner
buf.write("## %s \n" % owner)
buf.write(" * [%s](/%s): %s\n" % (alias, alias, title))
content = buf.getvalue()
if content == existing_content:
logging.info("Page '%s' does not need update\n", args.name)
sys.exit(0)
if args.output_only:
with open(args.output_only, "w") as fp:
fp.write(content)
logging.info("Wrote to %s\n", args.output_only)
sys.exit(0)
now = datetime.datetime.now()
cur.execute(
textwrap.dedent("""\
UPDATE "Notes" set
"content" = %s,
"lastchangeAt" = %s,
"updatedAt" = %s,
"title" = %s
WHERE id = %s;
"""),
(content, now, now, title, index_uuid))
if cur.rowcount != 1:
logging.error(
"Updated updated %d rows, expected 1: %s",
cur.rowcount, cur.statusmessage)
sys.exit(1)
logging.debug("update returned %s.", cur.statusmessage)
revId = uuid.uuid4()
cur.execute(
textwrap.dedent("""\
INSERT into "Revisions"
("id", "noteId", "content", "length", "createdAt", "updatedAt")
VALUES (%s, %s,%s,%s,%s,%s);
"""),
(str(revId), index_uuid, content, len(content), now, now))
if cur.rowcount != 1:
logging.error(
"Failed insert into Revisions (%d rows added): %s",
cur.rowcount, cur.statusmessage)
logging.debug("added revision %s for noteId %s", revId, index_uuid)
conn.commit()
logging.info("Updated page %s NoteId=%s RevisionId=%s\n",
args.name, index_uuid, revId)
cur.close()
sys.exit(0)
if __name__ == "__main__":
main()
#!/usr/bin/env python3
"""
Set author of a codi note.
$ codi-set-author [--note=|--owner=] new-user
new-user can be uuid or email
"""
import argparse
import base64
import datetime
import io
import logging
import os
import sys
import textwrap
import psycopg2
import uuid
def is_uuid(u):
try:
uuid.UUID(u)
except ValueError:
return False
# def uuid2b64(u):
# # codi strips off trailing '='.
# return base64.urlsafe_b64encode(uuid.UUID(u).bytes).decode('utf-8').rstrip("=")
#
#
def b642uuid(b):
# base64 values should be length of multiple of 4.
# you can just add = at the end, and python's b64decode
# will strip off the correct number.
return str(uuid.UUID(bytes=base64.urlsafe_b64decode(b + '=' * (4 - len(b) % 4))))
def get_user_by_input(info, cursor):
if is_uuid(info):
cursor.execute(textwrap.dedent("""
SELECT "Users".id, "Users".email, "Users".profileid
FROM "Users" WHERE "Users".id = %s"""),
(uuid.UUID(info),))
else:
cursor.execute(textwrap.dedent("""
SELECT "Users".id, "Users".email, "Users".profileid
FROM "Users"
WHERE ("Users".email = %s OR "Users".profileid = %s)"""),
(info, info))
if cursor.rowcount != 1:
logging.error("Found %d matches for user '%s':", cursor.rowcount, info)
for (cuuid, email, profileid) in cursor:
logging.error("uuid=%s email=%s profileid=%s", cuuid, email, profileid)
sys.exit(1)
return cursor.fetchone()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--dry-run', '-n', action='store_true')
parser.add_argument(
'-u', '--user', help='username to connect to postgres',
default="codimd")
parser.add_argument(
'-P', '--password', help='password for user',
default=os.environ.get('PGPASSWORD'))
parser.add_argument(
'-d', '--dbname', help='The name of the database',
default="codimd")
parser.add_argument(
'-H', '--hostname', help='connect to db on host HOSTNAME',
default="localhost"),
from_group = parser.add_mutually_exclusive_group(required=True)
from_group.add_argument(
'--owner', default=False, action='store_true',
help='owner as email or uuid')
from_group.add_argument(
'--page-uuids', default=False, action='store_true',
help='from-list are page uuids (or base64 encoded uuids)')
from_group.add_argument(
'--page-names', default=False, action='store_true',
help='from-list are page names (aliases)')
parser.add_argument('new_owner', help='new owner as email or uuid')
parser.add_argument('from_list', help='pages/notes or owner',
nargs="+")
args = parser.parse_args()
level = (logging.ERROR, logging.INFO, logging.DEBUG)[
min(args.verbose + 1, 2)]
logging.basicConfig(level=level)
conn = psycopg2.connect(
host=args.hostname, database=args.dbname, user=args.user,
password=args.password)
cur = conn.cursor()
(new_uuid, new_email, new_profileid) = get_user_by_input(args.new_owner, cur)
if args.owner:
# from-list is an owner by uuid or email.
if len(args.from_list) > 1:
logging.error("only a single owner supported (%s)", args.from_list)
sys.exit(1)
(owner_uuid, owner_email, owner_profileid) = get_user_by_input(args.from_list[0], cur)
cur.execute(textwrap.dedent("""
SELECT "Users".id, "Notes".id
FROM "Users" INNER JOIN "Notes"
ON "Notes"."ownerId" = "Users"."id"
WHERE "Users".id = %s"""),
(owner_uuid,))
page_uuids = []
for (cur_uuid, page_uuid) in cur:
page_uuids.append(page_uuid)
logging.info("Found %d pages owned by %s (uuid=%s)",
len(page_uuids), args.from_list[0], owner_uuid)
elif args.page_uuids:
# pages are base64 uuids or uuids
page_uuids = []
for page in args.from_list:
if is_uuid(page):
page_uuids.append(page)
else:
try:
page_uuids.append(b642uuid(page.rstrip("#")))
except ValueError:
logging.error("%s is not a uuid or base64 uuid", page)
sys.exit(1)
uuids = tuple(str(uuid.UUID(p)) for p in page_uuids)
cur.execute(textwrap.dedent("""
SELECT "Notes".id
FROM "Notes"
WHERE "Notes".id in %s"""),
(uuids,))
if cur.rowcount != len(page_uuids):
logging.error("you gave %d input ids, but %d exist as notes",
len(page_uuids), cur.rowcount)
sys.exit(1)
else:
# --page-names (aliases)
cur.execute(textwrap.dedent("""
SELECT "Notes".id, "Notes".alias
FROM "Notes"
WHERE "Notes".alias in %s"""),
(tuple(args.from_list),))
if cur.rowcount != len(args.from_list):
logging.error("you gave %d pages but %d exist",
len(args.from_list), cur.rowcount)
logging.error("Existing were:")
for (cuuid, alias) in cur:
logging.error("%s %s", cuuid, alias)
sys.exit(1)
page_uuids = [x[0] for x in cur]
if len(page_uuids) == 0:
logging.info("Nothing to do, 0 relevant page ids")
sys.exit(0)
owner_friendly = new_profileid if new_profileid else new_email
logging.info("Will change ownership of %d pages to %s (%s)",
len(page_uuids), owner_friendly, new_uuid)
cur.execute(textwrap.dedent("""
UPDATE "Notes" set "ownerId" = %s
WHERE id in %s
"""), (new_uuid, tuple(page_uuids),))
if cur.rowcount != len(page_uuids):
logging.error("Updated %d pages, but expected to update %d",
cur.rowcount, len(page_uuids))
cur.rollback()
sys.exit(1)
logging.info("Updated %d pages, setting owner to %s (%s)",
cur.rowcount, owner_friendly, new_uuid)
if args.dry_run:
logging.info("Dry run, rolling back\n")
conn.rollback()
else:
conn.commit()
conn.close()
sys.exit(0)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment