This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from datetime import date | |
from workalendar.usa import UnitedStates | |
# Get all holidays in the US | |
US_cal = UnitedStates() | |
US_cal.holidays(2022) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"basics": { | |
"name": "Khuyen Tran", | |
"label": "DevRel @ Prefect | Share 500+ daily tips on Python and data science", | |
"image": "", | |
"email": "khuyentran1476@gmail.com", | |
"phone": "6185146428", | |
"url": "https://github.com/khuyentran1401/", | |
"summary": "I'm the author of the book \"Efficient Python Tricks and Tools for Data Scientists\". I have written over 130 data science articles with over 10k followers on Medium. I have also posted over 400 tips on Python and data science on my website. \n\nMy current mission is to make open-source more accessible to the data science community. Reach out to me if you have or know any open-source libraries that are worth sharing. \n\nView my daily data science tips and blogs at https://mathdatasimplified.com.\nView my book at: https://khuyentran1401.github.io/Efficient_Python_tricks_and_tools_for_data_scientists", | |
"location": { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17:02:46.645 | INFO | prefect.agent - Submitting flow run '7fd49764-045f-4c68-a071-2470a1cf2070' | |
17:02:46.835 | INFO | prefect.infrastructure.docker-container - Creating Docker container 'zircon-ant'... | |
17:02:46.906 | INFO | prefect.infrastructure.docker-container - Docker container 'zircon-ant' has status 'created' | |
17:02:47.230 | INFO | prefect.agent - Completed submission of flow run '7fd49764-045f-4c68-a071-2470a1cf2070' | |
17:02:47.286 | INFO | prefect.infrastructure.docker-container - Docker container 'zircon-ant' has status 'running' | |
22:03:16.533 | ERROR | prefect.engine - Engine execution of flow run '7fd49764-045f-4c68-a071-2470a1cf2070' exited with unexpected exception | |
Traceback (most recent call last): | |
File "/usr/local/lib/python3.9/site-packages/prefect/client.py", line 1516, in read_flow_run | |
response = await self._client.get(f"/flow_runs/{flow_run_id}") | |
File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1751, in get |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from sqlalchemy import create_engine | |
engine = create_engine( | |
f"postgresql://{connection.user}:{connection.password}@{connection.host}/{connection.database}", | |
) | |
df.to_sql(name=name, con=engine, if_exists="replace", index=False) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from sqlalchemy import create_engine | |
engine = create_engine( | |
f"postgresql://{connection.user}:{connection.password}@{connection.host}/{connection.database}", | |
) | |
query = f'SELECT * FROM "{name}"' | |
df = pd.read_sql(query, con=engine) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This is Prefect 2.0b9 | |
import asyncio | |
from prefect import flow, task | |
@task | |
async def shout(number): | |
asyncio.sleep(1) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pathlib import Path | |
from prefect.deployments import Deployment, FlowScript | |
Deployment( | |
flow=FlowScript( | |
path=Path(__file__).parent / "train_model.py" | |
), | |
tags=["dev"], | |
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Traceback (most recent call last): | |
File "/Users/khuyen/prefect2-mlops-demo/venv/lib/python3.9/site-packages/alembic/script/base.py", line 233, in _catch_revision_errors | |
yield | |
File "/Users/khuyen/prefect2-mlops-demo/venv/lib/python3.9/site-packages/alembic/script/base.py", line 443, in _upgrade_revs | |
for script in reversed(list(revs)) | |
File "/Users/khuyen/prefect2-mlops-demo/venv/lib/python3.9/site-packages/alembic/script/revision.py", line 799, in iterate_revisions | |
revisions, heads = fn( | |
File "/Users/khuyen/prefect2-mlops-demo/venv/lib/python3.9/site-packages/alembic/script/revision.py", line 1454, in _collect_upgrade_revisions | |
current_revisions = self.get_revisions(lower) | |
File "/Users/khuyen/prefect2-mlops-demo/venv/lib/python3.9/site-packages/alembic/script/revision.py", line 527, in get_revisions |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from prefect import task, flow # new code | |
@task # new code | |
def get_authentication(): | |
... | |
@task # new code | |
def get_general_info_of_repos(auth: dict): | |
... |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import pandas as pd | |
from typing import List | |
import pandas as pd | |
from helper import load_config | |
from omegaconf import DictConfig | |
from pydash import py_ | |