generate_singer_messages.py

generate.py

"""
https://mimesis.name/en/master/schema.html
"""

import functools
import json
import sys
import typing as t

from mimesis import Field, Schema
from mimesis.locales import Locale
from singer_sdk import typing as th


class SingerField(th.Property):
    def __init__(self, *args, generator: t.Callable, **kwargs):
        super().__init__(*args, **kwargs)
        self.generator = generator


class SingerSchema(th.PropertiesList):
    def __init__(self, *fields: SingerField, examples: int = 10, **kwargs):
        super().__init__(*fields, **kwargs)
        self._schema = self._get_schema(examples=examples)

    def _get_schema(self, *, examples: int):
        return Schema(
            schema=lambda: {
                name: field.generator() for name, field in self.wrapped.items()
            },
            iterations=examples,
        )

    def create(self):
        return self._schema.create()


field = Field(locale=Locale.EN)


schema = SingerSchema(
    SingerField("id", th.StringType, generator=functools.partial(field, "increment")),
    SingerField(
        "name",
        th.StringType,
        generator=functools.partial(field, "person.name"),
    ),
)

sys.stdout.write(
    json.dumps({"type": "SCHEMA", "stream": "users", "schema": schema.to_dict()})
    + "\n",
)

for result in schema.create():
    sys.stdout.write(
        json.dumps({"type": "RECORD", "stream": "example", "record": result}) + "\n",
    )

requirements.txt

mimesis
singer-sdk

Example output:

{"type": "SCHEMA", "stream": "users", "schema": {"type": "object", "properties": {"id": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}}}
{"type": "RECORD", "stream": "example", "record": {"id": 1, "name": "Lucrecia"}}
{"type": "RECORD", "stream": "example", "record": {"id": 2, "name": "Lavera"}}
{"type": "RECORD", "stream": "example", "record": {"id": 3, "name": "Alphonse"}}
{"type": "RECORD", "stream": "example", "record": {"id": 4, "name": "Barney"}}
{"type": "RECORD", "stream": "example", "record": {"id": 5, "name": "Keeley"}}
{"type": "RECORD", "stream": "example", "record": {"id": 6, "name": "Jamison"}}
{"type": "RECORD", "stream": "example", "record": {"id": 7, "name": "Lajuana"}}
{"type": "RECORD", "stream": "example", "record": {"id": 8, "name": "Carol"}}
{"type": "RECORD", "stream": "example", "record": {"id": 9, "name": "Echo"}}
{"type": "RECORD", "stream": "example", "record": {"id": 10, "name": "Clifford"}}