EDA Markdown Generator is a production-ready Python library that automatically converts pandas DataFrames into comprehensive, publication-quality Exploratory Data Analysis (EDA) reports in Markdown format. It integrates seamlessly with the pandas-profiling library to extract statistical insights and formats them into structured, visually-enhance…

md_eda.py

import json
from functools import lru_cache
from typing import Any, Callable, Dict, List, Optional, Tuple

import pandas as pd
from data_profiling import ProfileReport

# ============================================================================
# CONSTANTS & CONFIGURATION
# ============================================================================

CORRELATION_THRESHOLDS = {
    "strong": 0.8,
    "moderate": 0.5,
    "weak": 0.3,
}

EMOJI_MAP = {
    "strong": "🔴",
    "moderate": "🟡",
    "weak": "🟢",
    "negligible": "⚪",
}

# Common table structures for variable analysis sections
BASIC_INFO_HEADERS = ["Property", "Value", "Property", "Value"]
COUNT_STATS_HEADERS = ["Metric", "Value", "Percentage"]
MISSING_STATS_HEADERS = ["Type", "Count", "Percentage"]
DESCRIPTIVE_STATS_HEADERS = ["Statistic", "Value", "Statistic", "Value"]
DISTRIBUTION_HEADERS = ["Measure", "Value"]
PERCENTILES_HEADERS = ["Percentile", "Value", "Percentile", "Value"]


# ============================================================================
# HELPER FUNCTIONS
# ============================================================================


def safe_pct(val: Any) -> str:
    """Format value as percentage safely."""
    if val is None:
        return "N/A"
    try:
        return f"{float(val) * 100:.2f}%"
    except (TypeError, ValueError):
        return "N/A"


def safe_val(val: Any, default: str = "N/A") -> str:
    """Return value as string or default safely."""
    if val is None:
        return default
    try:
        return str(val)
    except Exception:
        return default


def fmt_float(val: Any, decimals: int = 4) -> str:
    """Format float values safely with backticks."""
    if val is None:
        return "N/A"
    try:
        return f"`{float(val):.{decimals}f}`"
    except (ValueError, TypeError):
        return f"`{val}`"


@lru_cache(maxsize=16)
def _get_correlation_level(abs_val: float) -> str:
    """Get correlation strength level (cached)."""
    for level in ["strong", "moderate", "weak"]:
        if abs_val >= CORRELATION_THRESHOLDS[level]:
            return level
    return "negligible"


def corr_color(val: Optional[float]) -> str:
    """Return emoji + formatting based on correlation strength."""
    if val is None:
        return "N/A"

    abs_val = abs(val)
    level = _get_correlation_level(abs_val)
    emoji = EMOJI_MAP[level]
    bold = "**" if level in ["strong", "moderate"] else ""

    return f"{emoji} {bold}{val:.3f}{bold}"


# ============================================================================
# MARKDOWN BUILDER (OPTIMIZED)
# ============================================================================


class MarkdownBuilder:
    """Efficient markdown table and section builder with optimized string ops."""

    def __init__(self) -> None:
        self.lines: List[str] = []

    def add_section(self, title: str, level: int = 1) -> "MarkdownBuilder":
        """Add a markdown section header."""
        self.lines.append(f"{'#' * level} {title}")
        return self

    def add_blank(self) -> "MarkdownBuilder":
        """Add blank line."""
        self.lines.append("")
        return self

    def add_text(self, text: str) -> "MarkdownBuilder":
        """Add text line."""
        self.lines.append(text)
        return self

    def add_table(self, headers: List[str], rows: List[List[str]]) -> "MarkdownBuilder":
        """Add markdown table (optimized string joining)."""
        header_row = "| " + " | ".join(headers) + " |"
        separator = "|" + "|".join(["---"] * len(headers)) + "|"

        all_rows = [header_row, separator]
        all_rows.extend("| " + " | ".join(row) + " |" for row in rows)

        self.lines.extend(all_rows)
        return self

    def build(self) -> str:
        """Return final markdown string."""
        return "\n".join(self.lines)


# ============================================================================
# VARIABLE ANALYSIS HELPERS (Consolidated)
# ============================================================================


def _get_basic_info_row(data: Dict) -> List[str]:
    """Extract and format basic info row."""
    return [
        f"`{data.get('type', 'N/A')}`",
        f"`{data.get('ordering', 'N/A')}`",
        f"`{data.get('is_unique', 'N/A')}`",
        f"`{data.get('monotonicity', 'N/A')}`",
    ]


def _get_count_stats_rows(data: Dict) -> List[List[str]]:
    """Extract and format count statistics rows."""
    return [
        ["**Total Count (n)**", f"`{safe_val(data.get('n'))}`", "-"],
        [
            "**Distinct Values**",
            f"`{safe_val(data.get('n_distinct'))}`",
            safe_pct(data.get("p_distinct")),
        ],
        [
            "**Unique Values**",
            f"`{safe_val(data.get('n_unique'))}`",
            safe_pct(data.get("p_unique")),
        ],
    ]


def _get_missing_stats_rows(data: Dict, include_zeros: bool = False) -> List[List[str]]:
    """Extract and format missing/special values rows."""
    rows = [
        [
            "**Missing**",
            f"`{safe_val(data.get('n_missing'))}`",
            safe_pct(data.get("p_missing")),
        ],
    ]

    if include_zeros:
        rows.extend(
            [
                [
                    "**Zeros**",
                    f"`{safe_val(data.get('n_zeros'))}`",
                    safe_pct(data.get("p_zeros")),
                ],
                [
                    "**Infinite**",
                    f"`{safe_val(data.get('n_infinite'))}`",
                    safe_pct(data.get("p_infinite")),
                ],
            ]
        )

    return rows


def _get_descriptive_stats_rows(data: Dict) -> List[List[str]]:
    """Extract and format descriptive statistics rows."""
    return [
        [
            "**Mean**",
            f"`{safe_val(data.get('mean'))}`",
            "**Sum**",
            f"`{safe_val(data.get('sum'))}`",
        ],
        [
            "**Std Dev**",
            f"`{safe_val(data.get('std'))}`",
            "**Variance**",
            f"`{safe_val(data.get('variance'))}`",
        ],
        [
            "**Coeff. of Variation**",
            f"`{safe_val(data.get('cv'))}`",
            "**Median Abs. Dev.**",
            f"`{safe_val(data.get('mad'))}`",
        ],
    ]


def _get_percentiles_rows(data: Dict) -> List[List[str]]:
    """Extract and format percentile/range rows."""
    return [
        [
            "**Minimum**",
            f"`{safe_val(data.get('min'))}`",
            "**Maximum**",
            f"`{safe_val(data.get('max'))}`",
        ],
        [
            "**5th %**",
            f"`{safe_val(data.get('5%'))}`",
            "**Range**",
            f"`{safe_val(data.get('range'))}`",
        ],
        [
            "**Q1 (25%)**",
            f"`{safe_val(data.get('25%'))}`",
            "**IQR**",
            f"`{safe_val(data.get('iqr'))}`",
        ],
        ["**Median (50%)**", f"`{safe_val(data.get('50%'))}`", "", ""],
        ["**Q3 (75%)**", f"`{safe_val(data.get('75%'))}`", "", ""],
    ]


# ============================================================================
# MAIN ANALYSIS FUNCTIONS
# ============================================================================


def table_analysis(data: Dict) -> str:
    """Generate table-level analysis markdown."""
    md = MarkdownBuilder()
    md.add_section("Dimensions & Memory", 3).add_blank()
    md.add_text(f"**Rows (n)**: {safe_val(data.get('n'))}").add_blank()
    md.add_text(f"**Columns (n_var)**: {safe_val(data.get('n_var'))}").add_blank()

    md.add_section("Missing Data Summary", 3).add_blank()
    md.add_text(
        f"**Cells Missing**: {safe_val(data.get('n_cells_missing'))} ({safe_pct(data.get('p_cells_missing'))})"
    ).add_blank()
    md.add_text(
        f"**Vars with Missing**: {safe_val(data.get('n_vars_with_missing'))}"
    ).add_blank()
    md.add_text(
        f"**Vars All Missing**: {safe_val(data.get('n_vars_all_missing'))}"
    ).add_blank()

    # Column Types Distribution
    md.add_section("Column Types Distribution", 3).add_blank()
    types = data.get("types") or {}
    if types:
        type_items = list(types.items())
        for i, (t1, c1) in enumerate(type_items):
            if i % 2 == 0:
                t2, c2 = type_items[i + 1] if i + 1 < len(type_items) else ("", "")
                text = (
                    f"| **{t1}** | `{c1}` | **{t2}** | `{c2}` |"
                    if t2
                    else f"| **{t1}** | `{c1}` | - | - |"
                )
                md.add_text(text)
    else:
        md.add_text("*No type information available*")
    md.add_blank()

    # Data Quality
    md.add_section("Data Quality", 3).add_blank()
    md.add_text(
        f"**Duplicate Rows**: {safe_val(data.get('n_duplicates'))} ({safe_pct(data.get('p_duplicates'))})"
    ).add_blank()
    completeness = 100 - (data.get("p_cells_missing", 0) * 100)
    md.add_text(f"**Data Completeness**: {completeness:.2f}%").add_blank()

    return md.build()


def variable_analysis(data: Dict, var_type: str = "Numeric") -> str:
    """Generate variable-level analysis markdown (consolidated & polymorphic)."""
    md = MarkdownBuilder()

    # Common sections for all types
    md.add_section("Basic Information", 3).add_blank()
    md.add_table(BASIC_INFO_HEADERS, [_get_basic_info_row(data)]).add_blank()

    md.add_section("Count Statistics", 3).add_blank()
    md.add_table(COUNT_STATS_HEADERS, _get_count_stats_rows(data)).add_blank()

    md.add_section("Missing & Special Values", 3).add_blank()
    include_special = var_type == "Numeric"
    md.add_table(
        MISSING_STATS_HEADERS, _get_missing_stats_rows(data, include_special)
    ).add_blank()

    # Type-specific sections
    if var_type == "Numeric":
        md.add_section("Descriptive Statistics", 3).add_blank()
        md.add_table(
            DESCRIPTIVE_STATS_HEADERS, _get_descriptive_stats_rows(data)
        ).add_blank()

        md.add_section("Distribution Shape", 3).add_blank()
        md.add_table(
            DISTRIBUTION_HEADERS,
            [
                ["**Skewness**", f"`{safe_val(data.get('skewness'))}`"],
                ["**Kurtosis**", f"`{safe_val(data.get('kurtosis'))}`"],
            ],
        ).add_blank()

        md.add_section("Percentiles & Range", 3).add_blank()
        md.add_table(PERCENTILES_HEADERS, _get_percentiles_rows(data)).add_blank()

    elif var_type == "Categorical":
        md.add_section("Categories", 3).add_blank()
        cat_rows = [
            [f"`{cat}`", f"`{safe_val(data['word_counts'].get(cat))}`"]
            for cat in data.get("word_counts", {})
        ]
        md.add_table(["Value", "Count"], cat_rows).add_blank()

    return md.build()


def extract_extrema(correlations: List[Dict], variables: List[str]) -> Tuple:
    """Extract strongest positive and negative correlations efficiently (refactored)."""
    pairs: List[Tuple[str, str, float]] = []

    for i, row in enumerate(correlations):
        for j, var in enumerate(variables):
            if j > i:  # Upper triangle only
                pairs.append((variables[i], var, row[var]))

    if not pairs:
        return (None, None, -2.0), (None, None, 2.0)

    strongest_pos = max(pairs, key=lambda x: x[2])
    strongest_neg = min(pairs, key=lambda x: x[2])

    return strongest_pos, strongest_neg


def correlation_analysis(data: Dict) -> str:
    """Generate correlation analysis markdown."""
    auto = data.get("auto", [])
    if not auto:
        return "# ⚠️ No Correlation Data Available"

    variables = list(auto[0].keys())
    strongest_pos, strongest_neg = extract_extrema(auto, variables)

    md = MarkdownBuilder()
    md.add_section("Correlation Report").add_blank()
    md.add_section("Key Insights", 3).add_blank()

    md.add_table(
        ["Insight", "Details"],
        [
            [
                "**Strongest Positive**",
                f"`{strongest_pos[0]} ↔ {strongest_pos[1]}` = `{strongest_pos[2]:.3f}`",
            ],
            [
                "**Strongest Negative**",
                f"`{strongest_neg[0]} ↔ {strongest_neg[1]}` = `{strongest_neg[2]:.3f}`",
            ],
            ["**Variables Analyzed**", f"`{len(variables)}` ({', '.join(variables)})"],
            ["**Matrix Type**", "`Symmetric (Pearson)`"],
        ],
    ).add_blank()

    md.add_section("Upper Triangle", 3).add_blank()
    for i, row in enumerate(auto):
        for j, var in enumerate(variables):
            if j > i:
                val = row[var]
                md.add_text(f"- **{variables[i]} ↔ {var}**: {corr_color(val)}")
    md.add_blank()

    return md.build()


def sample_analysis(data: List[Dict]) -> str:
    """Generate sample data preview markdown."""
    if not isinstance(data, list) or not data:
        return "# ⚠️ No Sample Data Available"

    md = MarkdownBuilder()
    first_rows = data[0].get("data", [])
    variables = list(first_rows[0].keys()) if first_rows else []

    head_sample = next((s for s in data if s.get("id") == "head"), {})
    head_data = head_sample.get("data", [])

    if head_data:
        headers = ["#"] + variables
        rows = [
            [f"`{idx}`"] + [fmt_float(row.get(v)) for v in variables]
            for idx, row in enumerate(head_data, 1)
        ]
        md.add_table(headers, rows).add_blank()

    return md.build()


# ============================================================================
# DISPATCHER PATTERN (Single Maintenance Point)
# ============================================================================

VAR_TYPE_DISPATCH: Dict[str, str] = {
    "Categorical": "Categorical",
    "Text": "Text",
}


def main(df: pd.DataFrame, file_name: str = "eda_report.md") -> None:
    """Generate comprehensive EDA report."""
    profile = ProfileReport(
        df,
        title="Rapport EDA Détaillé",
        minimal=False,
        explorative=True,
        progress_bar=False,
    )

    data = json.loads(profile.to_json())
    md = MarkdownBuilder()

    md.add_section("Exploratory Data Analysis (EDA) Report")
    md.add_blank()
    md.add_section("Table Analysis").add_blank()
    md.add_text(table_analysis(data.get("table", {})))
    md.add_blank()

    alerts = data.get("alerts", [])
    if alerts:
        md.add_section("Alerts & Warnings").add_blank()
        for alert in alerts:
            md.add_text(f"- {alert}")
        md.add_blank()

    md.add_section("Analysis of Variables").add_blank()
    for var, var_data in data.get("variables", {}).items():
        md.add_section(f"Variable: {var}", 3).add_blank()
        var_type = var_data.get("type", "Numeric")
        # Use dispatch dict for type lookup
        analyzed_type = VAR_TYPE_DISPATCH.get(var_type, "Numeric")
        md.add_text(variable_analysis(var_data, analyzed_type))
        md.add_blank()

    md.add_section("Correlation Analysis").add_blank()
    md.add_text(correlation_analysis(data.get("correlations", {})))
    md.add_blank()

    md.add_section("Data Sample Preview").add_blank()
    md.add_text(sample_analysis(data.get("sample", [])))

    with open(file_name, "w", encoding="utf-8") as f:
        f.write(md.build())

requirements.txt

pandas
fg-data-profiling