convert_to_markdown.sh converts the wget-dump of an website into a collection of Markdown files.

convert_to_markdown.sh

#!/bin/sh

IN_DIR=~/Downloads/website_raw
OUT_DIR=content/pages
TITLE_STRING_SEP=' | '

XIDEL="xidel --input-format html"
PANDOC="pandoc -f html -t markdown_mmd"

[ -d $OUT_DIR ] || mkdir -p $OUT_DIR

for file in $(cd $IN_DIR ; find en de -name \*.html | grep -v '/node/')
do
  TITLE=$($XIDEL --extract '//title' "$IN_DIR/$file" | sed -e "s/$TITLE_STRING_SEP.*$//")
  TEXT=$($XIDEL --extract "//article" --output-format html "$IN_DIR/$file" | $PANDOC)
  #eval $($XIDEL -e 'TITLE:=//title' --output-format bash)
  LANG=${file:0:2}
  FILE_BASENAME=${file/.html/}
  SLUG=${FILE_BASENAME:3}
  MD_FILE=${FILE_BASENAME}.md

  CONTAINER=$(dirname "$OUT_DIR/$MD_FILE")
  [ -d $CONTAINER ] || mkdir -p $CONTAINER

	cat <<-EOT > "$OUT_DIR/$MD_FILE"
		title: $TITLE
		slug: $SLUG
		lang: $LANG

		$TEXT
EOT
done