CoreFoundation の CFStringTokenizer を使って（日本語でも）tokenize するやつ

tokenize.c

/* compile: cc tokenize.c -o tokenize -framework CoreFoundation */
/* usage: ./tokenize 今日は晴れでいい気分
/* 参考: https://stackoverflow.com/questions/6877659/how-to-get-an-array-of-sentences-using-cfstringtokenizer */

#include <stdio.h>
#include <CoreFoundation/CoreFoundation.h>

int main(int argc, char *argv[])
{
  if (argc != 2) {
    fprintf(stderr, "usage: %s String\n", argv[0]);
    exit(1);
  }
        
  CFStringRef string;
  // string = CFSTR("すもももももももものうち");
  string = CFStringCreateWithCString(kCFAllocatorDefault, argv[1], kCFStringEncodingUTF8);    
  CFLocaleRef locale = CFLocaleCopyCurrent();

  CFStringTokenizerRef tokenizer = CFStringTokenizerCreate(kCFAllocatorDefault,
                                                           string,
                                                           CFRangeMake(0, CFStringGetLength(string)),
                                                           kCFStringTokenizerUnitWord,
                                                           locale);

  CFStringTokenizerTokenType tokenType;

  while((tokenType = CFStringTokenizerAdvanceToNextToken(tokenizer)) != kCFStringTokenizerTokenNone) {
    CFRange tokenRange = CFStringTokenizerGetCurrentTokenRange(tokenizer);
    CFStringRef tokenValue = CFStringCreateWithSubstring(kCFAllocatorDefault,
                                                         string,
                                                         tokenRange);

    // CFShow(tokenValue);
    /* CFShowでは日本語は \u4eca のようになってしまうため printf する */
    /* 参考: https://stackoverflow.com/questions/9166291/converting-a-cfstringref-to-char */
    char *buffer;
    if ((buffer = (char *)CFStringGetCStringPtr(tokenValue, kCFStringEncodingUTF8)) != NULL) {
      printf("%s\n", buffer);
    } else {
      CFIndex length = CFStringGetLength(tokenValue);
      CFIndex maxSize = CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8) + 1;
      buffer = (char *)malloc(maxSize);
      CFStringGetCString(tokenValue, buffer, maxSize, kCFStringEncodingUTF8);
      printf("%s\n", buffer);
      free(buffer);
    }

    CFRelease(tokenValue);
  }

  CFRelease(tokenizer);
  CFRelease(locale);

  return 0;
}