Syncing Existing GitHub Repositories to Fabric with Semantic Link Labs | Sempy

Syncing Existing GitHub Repositories to Fabric with Semantic Link Labs | Sempy

# Author -Nalaka 14.03.2025 
!pip install PyGithub semantic-link-labs --q

# Import necessary libraries
from github import Github
import sempy.fabric as fabric
import sempy_labs as labs
import time

# GitHub Personal Access Token and connection ID
g = Github("xxx")
conn_id = "xx"
capacity_id = "xx"  # Ensure this is correct

# Fetch all repositories from GitHub
user = g.get_user()
repos = list(user.get_repos())  # Convert iterator to list

print(f"ℹ️ Total repositories found: {len(repos)}")

# List existing workspaces
existing_workspaces_df = fabric.list_workspaces()

# List existing Git connections
git_connections_df = labs.admin.list_git_connections()

# Initialize counters for summary
total_repos_processed = 0
workspaces_created = 0
workspaces_skipped = 0
workspaces_already_exist = 0
git_connections_created = 0
git_connections_skipped = 0
workspaces_synced_from_github = 0

# Iterate over each repository and create corresponding Fabric workspace
for repo in repos:
    total_repos_processed += 1
    try:
        if not repo.name.startswith("Bkp_"):
            print(f"⚠️ Skipping '{repo.name}', does not match 'Bkp_' prefix.")
            workspaces_skipped += 1
            continue  # Skip repos that don't start with 'Bkp_'
        
        workspace_name = repo.name.replace("Bkp_", "")  # Remove 'Bkp_' prefix
        print(f"Processing repo: {repo.name} -> Workspace name: {workspace_name}")

        # Check if the workspace already exists
        existing_workspace = existing_workspaces_df[existing_workspaces_df['Name'] == workspace_name]
        
        if not existing_workspace.empty:
            workspace_id = existing_workspace.iloc[0]['Id']
            print(f"ℹ️ Workspace '{workspace_name}' already exists with ID: {workspace_id}.")
            workspaces_already_exist += 1
            
            # Check if the workspace already has a Git connection
            if workspace_id in git_connections_df['Workspace Id'].values:
                print(f"⚠️ Workspace '{workspace_name}' already has a Git connection. Skipping creation and initialization.")
                git_connections_skipped += 1
                continue  # Skip to the next repository
            else:
                print(f"ℹ️ Workspace '{workspace_name}' does not have a Git connection. Proceeding to connect.")

        else:
            # Create workspace in Microsoft Fabric
            workspace_id = fabric.create_workspace(
                display_name=workspace_name,
                description=f"Workspace for {repo.name} synced from GitHub",
                capacity_id=capacity_id
            )
            print(f"🟢 Workspace '{workspace_name}' created with ID: {workspace_id}.")
            workspaces_created += 1

        # Connect workspace to the GitHub repository
        labs.connect_workspace_to_github(
            owner_name=user.login,
            repository_name=repo.name,
            branch_name="main",
            directory_name="/",
            connection_id=conn_id,
            workspace=workspace_id
        )
        print(f"🟢 Connected workspace '{workspace_name}' to GitHub repository '{repo.name}'.")
        git_connections_created += 1

        # Wait for Git connection to initialize
        time.sleep(10)  # Adjust as necessary
        try:
            commit_hash = labs.initialize_git_connection(workspace=workspace_id)
            print(f"🟢 Git connection initialized for '{workspace_name}'. Commit: {commit_hash}")
        except Exception as e:
            print(f"⚠️ Error initializing Git for '{workspace_name}': {e}")
            continue  # Move to the next repo

        # Automatically update workspace from Git
        try:
            update_status = labs.update_from_git(workspace=workspace_id, remote_commit_hash=commit_hash, conflict_resolution_policy='PreferRemote')
            print(f"🟢 Updated workspace '{workspace_name}' from Git. Status: {update_status}")
            workspaces_synced_from_github += 1
        except Exception as e:
            print(f"⚠️ Error updating workspace '{workspace_name}' from Git: {e}")

        # Sync all content from GitHub repo
        print(f"🟢 Workspace '{workspace_name}' setup completed. Moving to next repo.")

    except Exception as e:
        print(f"⚠️ Unexpected error processing '{repo.name}': {e}")

# Summary report
print(f"\n Summary Report:")  # Underlined title only
print(f"ℹ️ Total Repositories Processed: {total_repos_processed}")
print(f"🟢 Workspaces Created: {workspaces_created}")
print(f"⚠️ Workspaces Skipped (No 'Bkp_' prefix): {workspaces_skipped}")
print(f"ℹ️ Workspaces Already Existing: {workspaces_already_exist}")
print(f"🟢 Git Connections Created: {git_connections_created}")
print(f"⚠️ Git Connections Skipped (Already Existing): {git_connections_skipped}")
print(f"🟢 Workspaces Synced from GitHub: {workspaces_synced_from_github}")