TCF to sentence in plain text

convert.xq

xquery version "3.1";

declare namespace dta="http://www.dspin.de/data/textcorpus";

declare function local:load-dta($source, $target) {
    let $collection-uri := $target,
        $directory := $source,
        $pattern := "**/*",
        $mime-type := "application/xml",
        $preserve-structure := true()
    return
        xmldb:store-files-from-pattern($collection-uri, $directory, $pattern, $mime-type, $preserve-structure)
};

let $col := "/db/data/full"

for $corpus in collection($col)//dta:TextCorpus
let $name := ($corpus/base-uri() => tokenize("/"))[last()]
let $lemmas := $corpus//dta:lemma
let $token :=
        map:new(
            for $lemma in $lemmas
            return
                map:entry($lemma/string(@tokenIDs), $lemma)
        )
let $sentence := 
        for $sentence in $corpus//dta:sentence
        let $tokens := $sentence/@tokenIDs => tokenize(" ")
        let $sen := $tokens ! $token(.)
        return
            string-join($sen, " ")
let $data := string-join($sentence, "
")
            => replace("[^a-zA-ZüöäßÜÖÄ \n]", "")
let $binarydata :=  util:string-to-binary($data)
let $path := "/home/mathias/development/word2vec/texts/dta/"

return
    file:serialize-binary($binarydata, $path||replace($name, "\.tcf\.xml", ".sentences.txt"))