List fixed tokens of a file, those which are not alpnanumeric only

list-mixed-tokens.awk

#!/usr/bin/gawk -f

NF {
    for (i = 1; i <= NF; i++) {
        if ( $i !~ /^[[:alnum:]]+$/ ) {
            print $i;
        }
    }
}

Another usage: replace all groups of alphanumerics chars with 'a'.

cat *.mixed.tokens.txt | awk '{print $2}' | sed -E 's/[[:alnum:]-]+/a/g' | sort | uniq -c | sort | tac | head -n 100

 507817 a,
 353375 a.
  99448 a:
  95085 a
  59447 a;
  57525 "a
  50199 a)
  48918 a.a
  48288 a?
  47675 a,a
  47599 a",
  47215 (a
  39301 “a
  35448 a/a
  35136 a".
  34452 (a)
  33975 a),
  33481 a”,
  32763 a"
  30400 a).
  28052 "a"
  25635 “a”
  24726 (a),
  24025 a”
  22246 a!
  21300 a”.
  18659 a."
  18228 a,a%
  16376 a/a,
  14942 (a).
  14562 a.a,
  14464 'a
  14233 a.a.a/a
  13697 a,a.
  12917 a'
  10958 (a,
  10103 "a",
   9895 a.a/a,
   9190 “a”,
   8910 a/a.
   8888 'a'
   8579 a.a/a
   8159 a/a/a
   8074 a...
   7795 a.a.
   7649 a,a%.
   7601 a.a.a
   7468 a/a/a,
   7284 a,a,
   7014 a/a)
   6943 a,a%,
   6894 "a".
   6636 a.a,a
   6537 “a”.
   6373 a.a.a.a.a
   6312 #a
   6258 a…
   5909 a.”
   5890 a',
   5592 ‘a
   5402 a);
   5027 a/a/a.
   4643 "a,
   4519 a.a.a,
   4273 ‘a’
   4234 a’
   3766 a]
   3611 a.a.a.
   3521 a'a
   3325 a.a/a.
   3157 a:a
   2975 (a/a)
   2916 @a
   2880 [a
   2843 (a);
   2795 a.a;
   2722 a://a.a/a
   2663 a.a.a.a.a,
   2574 (a,a%)
   2537 a%
   2476 a'.
   2428 a):
   2343 (a):
   2282 a.a,a.
   2232 a,a%;
   2189 a"a.
   2176 a/a/a)
   2161 a/a/a).
   2081 (a,a%),
   2024 a’a
   1978 “a,
   1975 *a
   1956 [a]
   1937 a/a).
   1896 a/a/a;
   1892 a’,
   1888 a.a)
   1888 'a',
   1792 (@a)
   1788 a.a).