Skip to content

Instantly share code, notes, and snippets.

@anandology
Last active Feb 3, 2020
Embed
What would you like to do?
Lazily Load Python Modules
# import pandas as pd
from .heavy_modules import pandas as pd
# from google.cloud import bigquery
from .heavy_modules import google_cloud_bigquery as bigquery
def foo(path):
df = pd.read_csv(path)
...
"""
app.heavy_modules
~~~~~~~~~~~~~~~~~
This imports all heavy modules lazily.
"""
from lazy_import import lazy_import
numpy = lazy_import("numpy")
pandas = lazy_import("pandas")
google_cloud_storage = lazy_import("google.cloud.storage")
google_cloud_bigquery = lazy_import("google.cloud.bigquery")
import os
import web
# db_url = os.getenv("DATABASE_URL", "postgres:///elf")
# db = web.database(db_url)
#
# def get_sites():
# query = "..."
# result = db.query(query)
# ...
_db = None
def get_db():
global _db
if _db is None:
db_url = os.getenv("DATABASE_URL", "postgres:///elf")
_db = web.database(db_url)
return _db
def get_sites():
query = "..."
result = get_db().query(query)
...
# from statsmodels.tsa.holtwinters import SimpleExpSmoothing
def forecast(df):
"""Forecasts the future sales using the historical sales.
"""
# importing SimpleExpSmoothing inside the function to avoid loading it on startup
from statsmodels.tsa.holtwinters import SimpleExpSmoothing
model = SimpleExpSmoothing(df.quantity.values).fit()
....
"""
lazy_import
~~~~~~~~~~~
Utilitity to import a module lazily.
# import pandas as pd
pd = lazy_import("pandas")
"""
class LazyModule:
def __init__(self, name):
self.__dict__['_name'] = name
self.__dict__['_mod'] = None
def load(self):
if self._mod is None:
import importlib
self.__dict__['_mod'] = importlib.import_module(self._name)
return self._mod
def __getattr__(self, name):
return getattr(self.load(), name)
def __setattr__(self, name, value):
return setattr(self.load(), name, value)
def __delattr__(self, name):
return delattr(self.load(), name)
def __repr__(self):
return "<LazyModule: {}>".format(self._name)
def lazy_import(module_name):
"""Imports a module lazily.
The module is loaded when an attribute it is accessed from that module for the first time.
pd = lazy_import("pandas")
This only works with absolute imports and does not work with relative imports.
"""
return LazyModule(module_name)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment