Ceylon Web Runner: MemoStream

ceylonTypeTokenizer.ceylon

"Indicates an error during parsing."
shared class ParseError(String message) extends Exception(message){
}

class Token(shared Object type, shared String token, shared Integer index) {
    shared actual String string => "``token`` (``type``) at index ``index``";
}

"enumerates the different token types"
abstract class TokenType(shared actual String string)
        of dtAnd | dtOr | dtDot | dtComma | dtDColon | dtGt | dtLt
        | dtDigit | dtUpper | dtLower
        | dtQn | dtLsq | dtRsq | dtLbr | dtRbr | dtLparen | dtRparen
        | dtStar | dtPlus | dtRightArrow | dtEquals |dtIn | dtOut
        | dtEoi {}

object dtAnd extends TokenType("&") {}
object dtOr extends TokenType("|") {}
object dtDot extends TokenType(".") {}
object dtComma extends TokenType(",") {}
object dtDColon extends TokenType("::") {}
object dtGt extends TokenType(">") {}
object dtLt extends TokenType("<") {}
object dtDigit extends TokenType("digit") {}
object dtUpper extends TokenType("upper") {}
object dtLower extends TokenType("lower") {}
object dtEoi extends TokenType("<eoi>") {}

object dtQn extends TokenType("?") {}
object dtLsq extends TokenType("[") {}
object dtRsq extends TokenType("]") {}
object dtLbr extends TokenType("{") {}
object dtRbr extends TokenType("}") {}
object dtLparen extends TokenType("(") {}
object dtRparen extends TokenType(")") {}
object dtStar extends TokenType("*") {}
object dtPlus extends TokenType("+") {}
object dtRightArrow extends TokenType("->") {}
object dtEquals extends TokenType("=") {}

object dtIn extends TokenType("in") {}
object dtOut extends TokenType("out") {}

shared void tokenize() {

    function identifierCharacter(Character c)
        =>  c.letter || c.digit || c == '_';

    function identifier(TokenType type, {Character*} chars, Integer index)
        =>  let (string = String(chars.takeWhile(identifierCharacter)))
            [Token(type, string, index), string.size];

    [Token?] | [Token?, Integer] collecting({Character+} chars, Integer index) {
        value char = chars.first;
        if (char.whitespace) {
            return [null];
        }
        switch (char)
        case ('&') {
            return [Token(dtAnd, char.string, index)];
        }
        case ('|') {
            return [Token(dtOr, char.string, index)];
        }
        case ('.') {
            return [Token(dtDot, char.string, index)];
        }
        case (',') {
            return [Token(dtComma, char.string, index)];
        }
        case ('<') {
            return [Token(dtLt, char.string, index)];
        }
        case ('>') {
            return [Token(dtGt, char.string, index)];
        }
        case ('?') {
            return [Token(dtQn, char.string, index)];
        }
        case ('*') {
            return [Token(dtStar, char.string, index)];
        }
        case ('+') {
            return [Token(dtPlus, char.string, index)];
        }
        case ('[') {
            return [Token(dtLsq, char.string, index)];
        }
        case (']') {
            return [Token(dtRsq, char.string, index)];
        }
        case ('{') {
            return [Token(dtLbr, char.string, index)];
        }
        case ('}') {
            return [Token(dtRbr, char.string, index)];
        }
        case ('(') {
            return [Token(dtLparen, char.string, index)];
        }
        case (')') {
            return [Token(dtRparen, char.string, index)];
        }
        case ('=') {
            return [Token(dtEquals, char.string, index)];
        }
        case (':') {
            // check next is also :
            if (exists char2 = chars.rest.first) {
                if (char2 == ':') {
                    return [Token(dtDColon, String(chars.take(2)), index), 2];
                } else {
                    throw ParseError("tokenization error, expected ::, \
                                      not :``char2`` at index ``index``");
                }
            }
            throw ParseError("unexpected end of input");
        }
        case ('-') {
            // check next is >
            if (exists char2 = chars.rest.first) {
                if (char2 == '>') {
                    return [Token(dtRightArrow, String(chars.take(2)), index), 2];
                } else {
                    throw ParseError("tokenization error, expected ->, not -``char2`` \
                                      at index ``index``");
                }
            }
            throw ParseError("unexpected end of input");
        }
        else if ('0' <= char <= '9') {
            return [Token(dtDigit, char.string, index)];
        }
        else if (char.lowercase) {
            return identifier(dtLower, chars, index);
        }
        else if (char.uppercase) {
            return identifier(dtUpper, chars, index);
        }
        else if (char == '\\') {
            if (exists char2 = chars.getFromFirst(1)) {
                [Token, Integer] id;
                if (char2 == 'I') {
                    id = identifier(dtUpper, chars.skip(2), index);
                } else if (char2 == 'i') {
                    id = identifier(dtLower, chars.skip(2), index);
                } else {
                    throw ParseError("tokenization error, expected \\i or \\I, \
                                      not :\\``char2`` at index ``index``");
                }
                if (id[1] == 0) {
                    throw ParseError("unexpected character \
                                      ``chars.getFromFirst(2) else "<null>"`` \
                                      at index ``index+2``");
                }
                return [id[0], id[1]+2];
            }
            throw ParseError("unexpected end of input");
        } else {
            throw ParseError("unexpected character ``char`` at index ``index``");
        }
    }

    printAll {
        enhancedIterable("ceylon.language::Entry<String, Basic & Identifiable>")
            .transform(collecting)
            .coalesced;
        "\n";
    };
}

/*
    ceylon (lower) at index 0
    . (.) at index 6
    language (lower) at index 7
    :: (::) at index 15
    Entry (upper) at index 17
    < (<) at index 22
    String (upper) at index 23
    , (,) at index 29
    Basic (upper) at index 31
    & (&) at index 37
    Identifiable (upper) at index 39
    > (>) at index 51
*/

EnhancedIterable.ceylon

shared
interface EnhancedIterable<out Element, out Absent = Null>
        satisfies Iterable<Element, Absent>
        given Absent satisfies Null {

    shared
    Iterable<Result, Absent> transform<Result>(
            "Consume one or more elements, returning a result and the number of
             elements consumed."
            [Result, Integer=] collecting({Element+} elements, Integer index))
            => let (outerIterable = this) object
            satisfies Iterable<Result, Absent> {

        iterator() => object satisfies Iterator<Result> {
            variable Integer index = 0;
            variable Stream<Element> stream
                =   streamOf(outerIterable) else emptyStream;

            shared actual
            Result|Finished next() {
                if (!is EmptyStream current = stream) {
                    value tuple = collecting(current, index);
                    value result = tuple[0];
                    value consumed = tuple[1] else 1;

                    "At least one element must be consumed."
                    assert (consumed >= 1);
                    stream = current.skip(consumed);
                    index += consumed;
                    return result;
                }
                else {
                    return finished;
                }
            }
        };
    };
}

shared
EnhancedIterable<Element, Absent> enhancedIterable<Element, Absent>
        (Iterable<Element, Absent> delegate)
        given Absent satisfies Null => object
        satisfies EnhancedIterable<Element, Absent> {
    iterator() => delegate.iterator();
};

main.ceylon

shared void run() {
    value stream = streamOf(loop(1)((i) {
        print("generating from ``i``");
        if (i > 9) {
            return finished;
        }
        return i.successor;
    }));

    print(stream.first);
    print(stream.first);
    print(stream.skip(1).first);
    print(stream.skip(1).first);
    print(stream.skip(5).first);
    print(stream.skip(2).first);
    print(stream.skip(3).first);
    print(stream.skip(4).first);
    print(stream.skip(10).first);
}

/*
generating from 1
1
1
generating from 2
2
2
generating from 3
generating from 4
generating from 5
generating from 6
6
3
4
5
generating from 7
generating from 8
generating from 9
generating from 10
<null>
*/

memo.ceylon

interface Uninitialized of uninitialized {}

object uninitialized satisfies Uninitialized {}

Result() memo<Result>(Result() f) {
    variable Result|Uninitialized memo = uninitialized;
    return () => if (!is Uninitialized result = memo)
                 then result
                 else (memo = f());
}

MemoStream.ceylon

shared
interface Stream<out Element>
        of NonemptyStream<Element> | EmptyStream
        satisfies {Element*} {}

shared
interface EmptyStream of emptyStream satisfies Stream<Nothing> {}

shared
object emptyStream satisfies EmptyStream {
    iterator() => emptyIterator;
}

shared final
class NonemptyStream<out Element> satisfies Stream<Element> & {Element+} {

    Element car;
    Stream<Element>() cdr;

    new noMemo(Element first, Stream<Element> rest() => emptyStream) {
        this.car = first;
        this.cdr = rest;
    }

    shared new (Element first, Element | Finished next() => finished) {
        car = first;
        cdr = memo(() => switch (first = next())
                         case (is Finished) emptyStream
                         else NonemptyStream(first, next));
    }

    shared actual
    Element first => car;

    shared actual
    NonemptyStream<Element|Other> follow<Other>(Other head)
        =>  NonemptyStream<Element|Other>.noMemo(head, () => this);

    function self() => this;

    shared actual
    Iterator<Element> iterator() => object
            satisfies Iterator<Element> {

        variable Stream<Element>()? rest = self;

        shared actual
        Element|Finished next() {
            if (exists previous = rest) {
                value next = previous();
                if (!is EmptyStream next) {
                    rest = next.cdr;
                    return next.car;
                }
                rest = null;
            }
            return finished;
        }
    };

    shared actual
    Stream<Element> skip(Integer skipping) {
        if (skipping <= 0) {
            return this;
        }
        else {
            variable Stream<Element> result = this;
            variable value i=0;
            while (i++<skipping, !is EmptyStream current = result) {
                result = current.cdr();
            }
            return result;
        }
    }

    shared actual
    Stream<Element> rest
        =>  cdr();
}

shared
NonemptyStream<Element>|Absent streamOf<Element, Absent>
        (Iterable<Element, Absent> elements)
        given Absent satisfies Null {

    value next = elements.iterator().next;
    if (!is Finished first = next()) {
        return NonemptyStream(first, next);
    }
    assert (is Absent null);
    return null;
}

module.ceylon

module web_ide_script "1.0.0" {
    // Add module imports here
}

Click here to run this code online

(Actually, copy and paste locally and use the Dart or JVM backend...)

Why does streamOf return a Stream|Absent and not a MemoStream? (Or perhaps, why is such a function completely missing?)

If it returned MemoStream, you'd have to narrow to Stream before calling first. So, when you have a {T+} argument, it is much more convenient this way. If you want a MemoStream, you can do streamOf(...) else emptyStream.

FYI: I'm about to rename

• MemoStream --> Stream
• Stream --> NonemptyStream