search_for_repos.py

# Search GitHub API for repos using a Daft UDF and write the results.
def find_github_repos(query):
    res = []
    repos = github.search_repositories(query=query, sort="stars", order="desc")
    for repo in repos:
        res.append(
            {
                "name": repo.name,
                "owner": repo.owner.login,
            }
        )
    return res


queries = ["language:python", "language:rust", "language:javascript", "language:go"]
repos = daft.from_pydict({"queries": queries}).with_column(
    "repos",
    # Apply the UDF on the query column
    daft.col("queries").apply(
        find_github_repos,
        return_dtype=daft.DataType.struct(
            {
                "name": daft.DataType.string(),
                "owner": daft.DataType.string(),
            }
        ),
    ),
)
repos.show()
repos.write_parquet("repos")