list of words

p702.tokens.cpp.cpp

// p7e02.words.cpp
// juanfc 2023-11-24
// https://gist.github.com/juanfal/f13931517b6599093996884314d010cc

#include <iostream>
#include <array>
using namespace std;

const int N = 100;
typedef array<string,N> TListWords;

TListWords low(string s);
void printWords(TListWords l);

int main()
{
    printWords(low("a    number:    3 is _fine_"));
    return 0;
}

bool isWordChar(char c);

TListWords low(string s)
{
    TListWords r;
    int p = 0; // pos in TListWords
    bool inToken = true;
    for (int i = 0; i < s.length(); ++i)
        if (isWordChar(s[i])) {
            inToken = true;
            r[p] += s[i];
        } else {
            if (inToken)
                ++p;
            inToken = false;
        }
    return r;
}

void printWords(TListWords l)
{
    for (int i = 0; l[i].length() != 0; ++i)
        cout << '"' << l[i] << '"' << endl;
}


bool isWordChar(char c)
{
    return 'a' <= c and c <= 'z' or c == '_';
}

p7e02.words.cpp

// p7e02.words.cpp
// juanfc 2023-11-24
// https://gist.github.com/juanfal/f13931517b6599093996884314d010cc

#include <iostream>
#include <array>
using namespace std;

const int N = 100;
typedef array<string,N> TListWords;

TListWords low(string s);
void printWords(TListWords l);

int main()
{
    printWords(low("a    number:    3 is _fine_"));
    return 0;
}

bool isWordChar(char c);

TListWords low(string s)
{
    TListWords r;
    int p = 0; // pos in TListWords
    bool inToken = true;
    for (int i = 0; i < s.length(); ++i)
        if (isWordChar(s[i])) {
            inToken = true;
            r[p] += s[i];
        } else {
            if (inToken)
                ++p;
            inToken = false;
        }
    return r;
}

void printWords(TListWords l)
{
    for (int i = 0; l[i].length() != 0; ++i)
        cout << '"' << l[i] << '"' << endl;
}


bool isWordChar(char c)
{
    return 'a' <= c and c <= 'z' or c == '_';
}