Skip to content

Instantly share code, notes, and snippets.

@javier
Last active February 8, 2023 17:51
Show Gist options
  • Save javier/e7ca50d7c3bf4e414e2af1f8c7d7b556 to your computer and use it in GitHub Desktop.
Save javier/e7ca50d7c3bf4e414e2af1f8c7d7b556 to your computer and use it in GitHub Desktop.
Create notebook with energy time series data at 15 minutes interval. QuestDB demo
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "3193bdd7",
"metadata": {},
"outputs": [],
"source": [
"# pip install pandas questdb\n",
"import pandas as pd\n",
"\n",
"\n",
"df = pd.read_csv('https://data.open-power-system-data.org/time_series/2020-10-06/time_series_15min_singleindex.csv')\n",
"\n",
"\n",
"#upper bound is exclusive, so only 2018 and 2019 here\n",
"df = df.loc[df[\"utc_timestamp\"].between(\"2018-01-01\", \"2020-01-01\")]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "653e1852",
"metadata": {},
"outputs": [],
"source": [
"country_rows = []\n",
"country_codes = ['AT', 'BE', 'DE', 'HU', 'LU', 'NL']\n",
"common_columns = ['utc_timestamp']\n",
"base_column_names = ['load_actual_entsoe_transparency', 'load_forecast_entsoe_transparency']\n",
"\n",
"for country_code in country_codes:\n",
" local_column_names = [country_code + '_' + sub for sub in base_column_names]\n",
" country_columns = common_columns + local_column_names \n",
" country_df = df.filter(country_columns)\n",
" country_df.dropna(inplace=True)\n",
" country_df.columns = common_columns + base_column_names\n",
" country_df['country_code'] = country_code\n",
" country_rows.append(country_df)\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "a00540c8",
"metadata": {},
"outputs": [],
"source": [
"all_countries = pd.concat(country_rows).rename(columns={'utc_timestamp': 'timestamp', 'load_actual_entsoe_transparency': 'load_actual', 'load_forecast_entsoe_transparency': 'load_forecast'} )\n",
"all_countries.timestamp = pd.to_datetime(all_countries.timestamp, format=\"%Y-%m-%dT%H:%M:%SZ\")\n",
"all_countries.to_parquet('energy_15_mins.parquet.gzip', compression='gzip')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment