A small shell script for setting the area of a tablet through xinput/libinput, and making the setting permanent in xfconf (XFCE).

tablet-area.sh

#!/bin/sh
: <<'EOF'
Copyright © 2020 Torge Matthies <openglfreak@googlemail.com>
This work is free. You can redistribute it and/or modify it under the
terms of the Do What The Fuck You Want To Public License, Version 2,
as published by Sam Hocevar. See http://www.wtfpl.net/ for more details.
EOF

### Settings ###

device='XP-PEN STAR G640 Pen Pen (0)'

tablet_width=6
tablet_height=4
monitor_width=16
monitor_height=9

scale=3
scale_origin_x=0.5
scale_origin_y=0.2

### Code ###

calc() { printf '%s\n' "$*" | bc -l; }
fix_number() { printf '%s\n' "$*" | sed 's/^-\./-0./;s/^\./0./'; }
test_expr() { [ 'x1' = "x$(printf 'if (%s) 1\n' "$*" | bc -l)" ]; }
xfconf_device_name() { printf '%s\n' "$*" | sed 's/ /_/g;s/[^0-9A-Za-z_-]//g'; }

scale_ratio="$(calc "(($monitor_width) * ($tablet_height)) / (($tablet_width) * ($monitor_height))")"
if test_expr "($scale_ratio) < 1 && ($scale_ratio) > -1"; then
    scale_width="$(calc "($scale) * $scale_ratio")"
    scale_height="$(calc "($scale)")"
else
    scale_height="$(calc "($scale) * $scale_ratio")"
    scale_width="$(calc "($scale)")"
fi

offset_x="$(calc "(1 - ($scale_width)) * ($scale_origin_x)")"
offset_y="$(calc "(1 - ($scale_height)) * ($scale_origin_y)")"

offset_x="$(fix_number "$offset_x")"
offset_y="$(fix_number "$offset_y")"
scale_width="$(fix_number "$scale_width")"
scale_height="$(fix_number "$scale_height")"

xinput set-prop "$device" 'Coordinate Transformation Matrix' \
    "$scale_width" 0 "$offset_x" \
    0 "$scale_height" "$offset_y" \
    0 0 1 \
|| :

</dev/null >/dev/null 2>&1 \
xfconf-query -c pointers -p "/$(xfconf_device_name "$device")/Properties/Coordinate_Transformation_Matrix" --create -a -t double -s 0 \
    -t double -s "$scale_width" -t double -s 0 -t double -s "$offset_x" \
    -t double -s 0 -t double -s "$scale_height" -t double -s "$offset_y" \
    -t double -s 0 -t double -s 0 -t double -s 1 \
|| :