process_data.py

import json
import pandas as pd
from typing import List

import pandas as pd
from helper import load_config
from omegaconf import DictConfig
from prefect import flow, task
from prefect.tasks import task_input_hash
from pydash import py_


@task
def get_data(config: DictConfig):
    with open(config.data.raw, "r") as file:
        data = json.load(file)
    return data


@task(cache_key_fn=task_input_hash, cache_expiration=timedelta(days=1))
def filter_language(data: List[dict], language: str):
    """Only return repositories that are written in the specified language"""
    language = language.title()
    return py_(data).filter({"language": language}).value()