Download MSVC compiler/linker & Windows SDK without installing full Visual Studio

!portable-msvc.md

Forked from: https://gist.github.com/mmozeiko/7f3162ec2988e81e56d5c4e22cde9977

This downloads standalone 64-bit MSVC compiler, linker & other tools, also headers/libraries from Windows SDK into portable folder, without installing Visual Studio. Has bare minimum components - no UWP/Store/WindowsRT stuff, just files & tools for 64-bit native desktop app development.

Run python.exe portable-msvc.py and it will download output into msvc folder. By default it will download latest available MSVC & Windows SDK
(MANIFEST_URL = "https://aka.ms/vs/17/release/channel")
(https://learn.microsoft.com/en-us/visualstudio/releases/2022/release-history)

You can list available versions with python.exe portable-msvc.py --show-versions and then pass versions you want with --msvc-version and --sdk-version arguments.

To use cl.exe/link.exe from output folder, first run setup.bat - after that PATH/INCLUDE/LIB env variables will be setup to use all the tools as usual. You can also use clang-cl.exe with these includes & libraries.

To use clang-cl.exe without running setup.bat, pass extra /winsysroot msvc argument (msvc is folder name where output is stored).

portable-msvc.py

#!/usr/bin/env python3

import io
import os
import sys
import json
import shutil
import hashlib
import zipfile
import tempfile
import argparse
import subprocess
import urllib.request
from pathlib import Path

OUTPUT = Path("msvc") # output folder

# other architectures may work or may not - not really tested
HOST   = "x64" # or x86
TARGET = "x64" # or x86, arm, arm64

MANIFEST_URL = "https://aka.ms/vs/17/release/channel"


def download(url):
  with urllib.request.urlopen(url) as res:
    return res.read()

def download_progress(url, check, name, f):
  data = io.BytesIO()
  with urllib.request.urlopen(url) as res:
    total = int(res.headers["Content-Length"])
    size = 0
    while True:
      block = res.read(1<<20)
      if not block:
        break
      f.write(block)
      data.write(block)
      size += len(block)
      perc = size * 100 // total
      print(f"\r{name} ... {perc}%", end="")
  print()
  data = data.getvalue()
  digest = hashlib.sha256(data).hexdigest()
  if check.lower() != digest:
    exit(f"Hash mismatch for f{pkg}")
  return data

# super crappy msi format parser just to find required .cab files
def get_msi_cabs(msi):
  index = 0
  while True:
    index = msi.find(b".cab", index+4)
    if index < 0:
      return
    yield msi[index-32:index+4].decode("ascii")

def first(items, cond):
  return next(item for item in items if cond(item))
  

### parse command-line arguments

ap = argparse.ArgumentParser()
ap.add_argument("--show-versions", const=True, action="store_const", help="Show available MSVC and Windows SDK versions")
ap.add_argument("--accept-license", const=True, action="store_const", help="Automatically accept license")
ap.add_argument("--msvc-version", help="Get specific MSVC version")
ap.add_argument("--sdk-version", help="Get specific Windows SDK version")
args = ap.parse_args()


### get main manifest

manifest = json.loads(download(MANIFEST_URL))


### download VS manifest

vs = first(manifest["channelItems"], lambda x: x["id"] == "Microsoft.VisualStudio.Manifests.VisualStudio")
payload = vs["payloads"][0]["url"]

vsmanifest = json.loads(download(payload))


### find MSVC & WinSDK versions

packages = {}
for p in vsmanifest["packages"]:
  packages.setdefault(p["id"].lower(), []).append(p)

msvc = {}
sdk = {}

for pid,p in packages.items():
  if pid.startswith("Microsoft.VisualStudio.Component.VC.".lower()) and pid.endswith(".x86.x64".lower()):
    pver = ".".join(pid.split(".")[4:6])
    if pver[0].isnumeric():
      msvc[pver] = pid
  elif pid.startswith("Microsoft.VisualStudio.Component.Windows10SDK.".lower()) or \
       pid.startswith("Microsoft.VisualStudio.Component.Windows11SDK.".lower()):
    pver = pid.split(".")[-1]
    if pver.isnumeric():
      sdk[pver] = pid

if args.show_versions:
  print("MSVC versions:", " ".join(sorted(msvc.keys())))
  print("Windows SDK versions:", " ".join(sorted(sdk.keys())))
  exit(0)

msvc_ver = args.msvc_version or max(sorted(msvc.keys()))
sdk_ver = args.sdk_version or max(sorted(sdk.keys()))

if msvc_ver in msvc:
  msvc_pid = msvc[msvc_ver]
  msvc_ver = ".".join(msvc_pid.split(".")[4:-2])
else:
  exit(f"Unknown MSVC version: f{args.msvc_version}")

if sdk_ver in sdk:
  sdk_pid = sdk[sdk_ver]
else:
  exit(f"Unknown Windows SDK version: f{args.sdk_version}")

print(f"Downloading MSVC v{msvc_ver} and Windows SDK v{sdk_ver}")


### agree to license

tools = first(manifest["channelItems"], lambda x: x["id"] == "Microsoft.VisualStudio.Product.BuildTools")
resource = first(tools["localizedResources"], lambda x: x["language"] == "en-us")
license = resource["license"]

if not args.accept_license:
  accept = input(f"Do you accept Visual Studio license at {license} [Y/N] ? ")
  if not accept or accept[0].lower() != "y":
    exit(0)

OUTPUT.mkdir(exist_ok=True)
total_download = 0

### download MSVC

msvc_packages = [
  # MSVC binaries
  f"microsoft.vc.{msvc_ver}.tools.host{HOST}.target{TARGET}.base",
  f"microsoft.vc.{msvc_ver}.tools.host{HOST}.target{TARGET}.res.base",
  # MSVC headers
  f"microsoft.vc.{msvc_ver}.crt.headers.base",
  # MSVC libs
  f"microsoft.vc.{msvc_ver}.crt.{TARGET}.desktop.base",
  f"microsoft.vc.{msvc_ver}.crt.{TARGET}.store.base",
  # MSVC runtime source
  f"microsoft.vc.{msvc_ver}.crt.source.base",
  # ASAN
  f"microsoft.vc.{msvc_ver}.asan.headers.base",
  f"microsoft.vc.{msvc_ver}.asan.{TARGET}.base",
  # MSVC redist
  #f"microsoft.vc.{msvc_ver}.crt.redist.x64.base",
]

for pkg in msvc_packages:
  p = first(packages[pkg], lambda p: p.get("language") in (None, "en-US"))
  for payload in p["payloads"]:
    with tempfile.TemporaryFile() as f:
      data = download_progress(payload["url"], payload["sha256"], pkg, f)
      total_download += len(data)
      with zipfile.ZipFile(f) as z:
        for name in z.namelist():
          if name.startswith("Contents/"):
            out = OUTPUT / Path(name).relative_to("Contents")
            out.parent.mkdir(parents=True, exist_ok=True)
            out.write_bytes(z.read(name))


### download Windows SDK

sdk_packages = [
  # Windows SDK tools (like rc.exe & mt.exe)
  f"Windows SDK for Windows Store Apps Tools-x86_en-us.msi",
  # Windows SDK headers
  f"Windows SDK for Windows Store Apps Headers-x86_en-us.msi",
  f"Windows SDK Desktop Headers x86-x86_en-us.msi",
  # Windows SDK libs
  f"Windows SDK for Windows Store Apps Libs-x86_en-us.msi",
  f"Windows SDK Desktop Libs {TARGET}-x86_en-us.msi",
  # CRT headers & libs
  f"Universal CRT Headers Libraries and Sources-x86_en-us.msi",
  # CRT redist
  #"Universal CRT Redistributable-x86_en-us.msi",
]

with tempfile.TemporaryDirectory() as d:
  dst = Path(d)

  sdk_pkg = packages[sdk_pid][0]
  sdk_pkg = packages[first(sdk_pkg["dependencies"], lambda x: True).lower()][0]

  msi = []
  cabs = []

  # download msi files
  for pkg in sdk_packages:
    payload = first(sdk_pkg["payloads"], lambda p: p["fileName"] == f"Installers\\{pkg}")
    msi.append(dst / pkg)
    with open(dst / pkg, "wb") as f:
      data = download_progress(payload["url"], payload["sha256"], pkg, f)
      total_download += len(data)
      cabs += list(get_msi_cabs(data))

  # download .cab files
  for pkg in cabs:
    payload = first(sdk_pkg["payloads"], lambda p: p["fileName"] == f"Installers\\{pkg}")
    with open(dst / pkg, "wb") as f:
      download_progress(payload["url"], payload["sha256"], pkg, f)

  print("Unpacking msi files...")

  # run msi installers
  for m in msi:
    subprocess.check_call(["msiexec.exe", "/a", m, "/quiet", "/qn", f"TARGETDIR={OUTPUT.resolve()}"])


### versions

msvcv = list((OUTPUT / "VC/Tools/MSVC").glob("*"))[0].name
sdkv = list((OUTPUT / "Windows Kits/10/bin").glob("*"))[0].name


# place debug CRT runtime files into MSVC folder (not what real Visual Studio installer does... but is reasonable)

dst = OUTPUT / "VC/Tools/MSVC" / msvcv / f"bin/Host{HOST}/{TARGET}"

with tempfile.TemporaryDirectory() as d:
  d = Path(d)
  pkg = "microsoft.visualcpp.runtimedebug.14"
  dbg = first(packages[pkg], lambda p: p["chip"] == HOST)
  for payload in dbg["payloads"]:
    name = payload["fileName"]
    with open(d / name, "wb") as f:
      data = download_progress(payload["url"], payload["sha256"], f"{pkg}/{name}", f)
      total_download += len(data)
  msi = d / first(dbg["payloads"], lambda p: p["fileName"].endswith(".msi"))["fileName"]

  with tempfile.TemporaryDirectory() as d2:
    subprocess.check_call(["msiexec.exe", "/a", str(msi), "/quiet", "/qn", f"TARGETDIR={d2}"])
    for f in first(Path(d2).glob("System*"), lambda x: True).iterdir():
      f.replace(dst / f.name)

# download DIA SDK and put msdia140.dll file into MSVC folder

with tempfile.TemporaryDirectory() as d:
  d = Path(d)
  pkg = "microsoft.visualc.140.dia.sdk.msi"
  dia = packages[pkg][0]
  for payload in dia["payloads"]:
    name = payload["fileName"]
    with open(d / name, "wb") as f:
      data = download_progress(payload["url"], payload["sha256"], f"{pkg}/{name}", f)
      total_download += len(data)
  msi = d / first(dia["payloads"], lambda p: p["fileName"].endswith(".msi"))["fileName"]

  with tempfile.TemporaryDirectory() as d2:
    subprocess.check_call(["msiexec.exe", "/a", str(msi), "/quiet", "/qn", f"TARGETDIR={d2}"])

    if HOST == "x86": msdia = "msdia140.dll"
    elif HOST == "x64": msdia = "amd64/msdia140.dll"
    else: exit("unknown")

    src = Path(d2) / "Program Files" / "Microsoft Visual Studio 14.0" / "DIA SDK" / "bin" / msdia
    src.replace(dst / "msdia140.dll")


### cleanup

shutil.rmtree(OUTPUT / "Common7", ignore_errors=True)
for f in ["Auxiliary", f"lib/{TARGET}/store", f"lib/{TARGET}/uwp"]:
  shutil.rmtree(OUTPUT / "VC/Tools/MSVC" / msvcv / f)
for f in OUTPUT.glob("*.msi"):
  f.unlink()
for f in ["Catalogs", "DesignTime", f"bin/{sdkv}/chpe", f"Lib/{sdkv}/ucrt_enclave"]:
  shutil.rmtree(OUTPUT / "Windows Kits/10" / f, ignore_errors=True)
for arch in ["x86", "x64", "arm", "arm64"]:
  if arch != TARGET:
    shutil.rmtree(OUTPUT / "VC/Tools/MSVC" / msvcv / f"bin/Host{arch}", ignore_errors=True)
    shutil.rmtree(OUTPUT / "Windows Kits/10/bin" / sdkv / arch)
    shutil.rmtree(OUTPUT / "Windows Kits/10/Lib" / sdkv / "ucrt" / arch)
    shutil.rmtree(OUTPUT / "Windows Kits/10/Lib" / sdkv / "um" / arch)


### setup.bat

SETUP = f"""@echo off

set ROOT=%~dp0

set MSVC_VERSION={msvcv}
set MSVC_HOST=Host{HOST}
set MSVC_ARCH={TARGET}
set SDK_VERSION={sdkv}
set SDK_ARCH={TARGET}

set MSVC_ROOT=%ROOT%VC\\Tools\\MSVC\\%MSVC_VERSION%
set SDK_INCLUDE=%ROOT%Windows Kits\\10\\Include\\%SDK_VERSION%
set SDK_LIBS=%ROOT%Windows Kits\\10\\Lib\\%SDK_VERSION%

set VCToolsInstallDir=%MSVC_ROOT%\\
set PATH=%MSVC_ROOT%\\bin\\%MSVC_HOST%\\%MSVC_ARCH%;%ROOT%Windows Kits\\10\\bin\\%SDK_VERSION%\\%SDK_ARCH%;%ROOT%Windows Kits\\10\\bin\\%SDK_VERSION%\\%SDK_ARCH%\\ucrt;%PATH%
set INCLUDE=%MSVC_ROOT%\\include;%SDK_INCLUDE%\\ucrt;%SDK_INCLUDE%\\shared;%SDK_INCLUDE%\\um;%SDK_INCLUDE%\\winrt;%SDK_INCLUDE%\\cppwinrt
set LIB=%MSVC_ROOT%\\lib\\%MSVC_ARCH%;%SDK_LIBS%\\ucrt\\%SDK_ARCH%;%SDK_LIBS%\\um\\%SDK_ARCH%
"""

(OUTPUT / "setup.bat").write_text(SETUP)

print(f"Total downloaded: {total_download>>20} MB")
print("Done!")

refactored_portable_msvc.py

# ruff: noqa: E501 F541 E701 F401 E731
# E501 (line too loong)
# F541 (f-string without any placeholders)
# E701 (Multiple statements on one line (colon))
# F401 (package imported but not used)
import argparse
import collections
import hashlib
import io
import json
import random
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.request
import zipfile
from pathlib import Path
from typing import Tuple

import colorama
import requests
from colorama import Fore
from tqdm import tqdm

colorama.init(autoreset=True)

Total_download_bytes_count = 0


def main():
    package_store_dir = Path("packages")

    manifest = parse_channel("https://aka.ms/vs/17/release/channel", Path("channel"), Path("VisualStudio.vsman"))

    msvc_ver, sdk_pkg_id = parse_manifest(
        manifest
    )  # ..           '14.40.17.10', 'microsoft.visualstudio.component.windows11sdk.26100'
    out_dir = Path(f"vs2022_{msvc_ver[6:]}")  # e.g. msvc_ver[6:] =       17.10
    print(f"\nOUTPUT:\n" f"\t{out_dir.resolve()}")

    download_msvc(
        out_dir,  # ..           Path('vs2022_17.10')
        package_store_dir,  # .. Path('packages')
        msvc_ver,  # ..          "14.40.17.10"
        manifest,
    )

    download_sdk(
        out_dir,  # ..            Path('vs2022_17.10')
        package_store_dir,  # ..  Path('packages')
        sdk_pkg_id,  # ..         'microsoft.visualstudio.component.windows11sdk.26100'
        manifest,
    )

    host = "x64"
    for target in ("x64", "x86"):
        download_and_place_runtimedebug(
            host,
            target,
            out_dir,
            package_store_dir / "runtimedebug",  #
            manifest,
        )
        download_and_place_msdia140(
            host,
            target,
            out_dir,
            package_store_dir / "dia_sdk",  #
            manifest,
        )
        write_setup_bat(host, target, out_dir)

    cleanup(out_dir)  # remove "arm" dirs

    print(
        f"\nTotal downloaded: {Total_download_bytes_count >> 20} MB"  #
        f"\nDone!"
    )
    sys.exit(0)


def parse_channel(channel_url, channel_file, visual_studio_vsman) -> dict:
    print(f"local channel {channel_file.resolve()}" if channel_file.exists() else "downloading remote channel...")
    channel: dict = json.loads(
        channel_file.read_text(encoding="utf-8")
        if channel_file.exists()  #
        else _download(channel_url, channel_file)
    )

    manifest_url = _first(  #
        channel["channelItems"], lambda x: x["id"] == "Microsoft.VisualStudio.Manifests.VisualStudio"
    )["payloads"][0]["url"]  # manifest_sha256 = _vs["payloads"][0]["sha256"] # ??? wrong sha256 ???

    print(
        f"local manifest {visual_studio_vsman.resolve()}"
        if visual_studio_vsman.exists()
        else "downloading remote VisualStudio.vsman"
    )

    manifest = json.loads(
        visual_studio_vsman.read_text(encoding="utf-8")
        if visual_studio_vsman.exists()
        else _download(manifest_url, visual_studio_vsman)
    )

    packages = collections.defaultdict(list)  # {"id":[]}
    for pkg in manifest["packages"]:
        packages[pkg["id"].lower()].append(pkg)

    return packages


def parse_manifest(manifest):
    _vc_pid: str
    sdk_pid: str
    _vc_pid, sdk_pid = _get_latest(manifest)

    print(
        f"\nDownloading\n"
        f"\t{_vc_pid.replace("microsoft.visualstudio.component.", "")}\n"  # .. microsoft.visualstudio.component.vc.14.40.17.10.x86.x64
        f"\t{sdk_pid.replace("microsoft.visualstudio.component.", "")}\n"
    )  # microsoft.visualstudio.component.windows11sdk.26100

    vc_ver = ".".join(_vc_pid.split(".")[4:-2])  # e.g. 14.40.17.10

    return vc_ver, sdk_pid


def _get_latest(manifest: dict) -> Tuple[str, str]:
    vc_dict_ = {}  # {pver, pid}, e.g. '14.40' => 'microsoft.visualstudio.component.vc.14.40.17.10.x86.x64'
    sdk_dict = {}  # {pver, pid}, e.g. '26100' => 'microsoft.visualstudio.component.windows11sdk.26100'

    pid: str

    for pid in manifest.keys():
        if pid.startswith("microsoft.visualstudio.component.vc") and pid.endswith(".x86.x64"):
            ver = ".".join(pid.split(".")[4:6])
            if ver[0].isnumeric():
                vc_dict_[ver] = pid
        elif pid.startswith("microsoft.visualstudio.component.windows10sdk.") or pid.startswith(
            "microsoft.visualstudio.component.windows11sdk."
        ):
            ver = pid.split(".")[-1]
            if ver.isnumeric():
                sdk_dict[ver] = pid

    print(f'\nAvaliable MSVC versions: (total {len(vc_dict_)})\n\t{" ".join(sorted(vc_dict_.keys()))}')
    print(f'Avaliable Windows SDK versions: (total {len(sdk_dict)})\n\t{" ".join(sorted(sdk_dict.keys()))}')

    max_vc_ver_ = max(sorted(vc_dict_.keys()))
    max_sdk_ver = max(sorted(sdk_dict.keys()))

    return vc_dict_[max_vc_ver_], sdk_dict[max_sdk_ver]


def download_msvc(
    output_dir: Path,
    vsix_save_to_dir: Path,
    ver: str,  # e.g. ver = 14.40.17.10
    manifest: dict,
):
    global Total_download_bytes_count

    output_dir.mkdir(parents=True, exist_ok=True)
    vsix_save_to_dir.mkdir(parents=True, exist_ok=True)

    print("\n### MSVC")
    known_msvc_pkg_ids_to_download = [
        # MSVC binaries
        f"microsoft.vc.{ver}.tools.hostx64.targetx86.base",
        f"microsoft.vc.{ver}.tools.hostx64.targetx64.base",
        f"microsoft.vc.{ver}.tools.hostx64.targetx86.res.base",
        f"microsoft.vc.{ver}.tools.hostx64.targetx64.res.base",
        # MSVC headers
        f"microsoft.vc.{ver}.crt.headers.base",
        # MSVC libs
        f"microsoft.vc.{ver}.crt.x86.desktop.base",
        f"microsoft.vc.{ver}.crt.x64.desktop.base",
        f"microsoft.vc.{ver}.crt.x86.store.base",
        f"microsoft.vc.{ver}.crt.x64.store.base",
        # MSVC runtime source
        f"microsoft.vc.{ver}.crt.source.base",
        # ASAN
        f"microsoft.vc.{ver}.asan.headers.base",
        f"microsoft.vc.{ver}.asan.x86.base",
        f"microsoft.vc.{ver}.asan.x64.base",
        # ATL
        f"microsoft.vc.{ver}.atl.headers.base",
        f"microsoft.vc.{ver}.atl.source.base",
        f"microsoft.vc.{ver}.atl.x86.base",
        f"microsoft.vc.{ver}.atl.x64.base",
        # f"microsoft.vc.{version}.atl.x86.spectre.base",
        # f"microsoft.vc.{version}.atl.x64.spectre.base",
        # MFC
        f"microsoft.vc.{ver}.mfc.headers.base",
        f"microsoft.vc.{ver}.mfc.source.base",
        f"microsoft.vc.{ver}.mfc.x86.base",
        f"microsoft.vc.{ver}.mfc.x64.base",
        # f"microsoft.vc.{version}.mfc.x86.spectre.base",
        # f"microsoft.vc.{version}.mfc.x64.spectre.base",
        f"microsoft.vc.{ver}.mfc.x86.debug.base",
        f"microsoft.vc.{ver}.mfc.x64.debug.base",
        # f"microsoft.vc.{version}.mfc.x86.debug.spectre.base",
        # f"microsoft.vc.{version}.mfc.x64.debug.spectre.base",
        f"microsoft.vc.{ver}.mfc.mbcs.base",  # x86 only
        # f"microsoft.vc.{version}.mfc.mbcs.spectre.base",  # x86 only
        f"microsoft.vc.{ver}.mfc.mbcs.debug.base",  # x86 only
        # f"microsoft.vc.{version}.mfc.mbcs.debug.spectre.base",  # x86 only
        f"microsoft.vc.{ver}.mfc.mbcs.x64.base",  # x64 only
        # f"microsoft.vc.{version}.mfc.mbcs.x64.spectre.base",  # x64 only
        f"microsoft.vc.{ver}.mfc.mbcs.x64.debug.base",  # x64 only

        # f"microsoft.vc.{version}.mfc.mbcs.x64.debug.spectre.base",  # x64 only
        # f"microsoft.vc.{ver}.mfc.redist.x86.base",
        # f"microsoft.vc.{ver}.mfc.redist.x64.base",
        # f"microsoft.vc.{ver}.mfc.redist.x86.spectre.base",
        # f"microsoft.vc.{ver}.mfc.redist.x64.spectre.base",
        
        # REDIST
        # f"microsoft.vc.{ver}.crt.redist.x86.base",
        # f"microsoft.vc.{ver}.crt.redist.x64.base",
    ]

    for a_msvc_pkg_id in known_msvc_pkg_ids_to_download:
        pkg_eng = _first(manifest[a_msvc_pkg_id], lambda x: x.get("language") in (None, "en-US"))

        for payload in pkg_eng["payloads"]:
            vsix_file_save_to = Path(vsix_save_to_dir) / "msvc" / payload["fileName"]
            print()
            print(vsix_file_save_to)
            print(payload["url"])

            Total_download_bytes_count += len(_download_with_sha256(payload["url"], payload["sha256"], vsix_file_save_to))

            with zipfile.ZipFile(vsix_file_save_to) as z:  # unzip *.vsix files
                for name in z.namelist():
                    if name.startswith("Contents/"):
                        out = output_dir / Path(name).relative_to("Contents")
                        print(f"\t{out}")
                        out.parent.mkdir(parents=True, exist_ok=True)
                        out.write_bytes(z.read(name))


def download_sdk(
    output_dir: Path,
    sdk_save_to_dir: Path,
    win_sdk_pkg_id_chosen_by_user: str,  #  e.g. microsoft.visualstudio.component.windows11sdk.26100
    manifest: dict,
):
    global Total_download_bytes_count

    output_dir.mkdir(parents=True, exist_ok=True)
    sdk_save_to_dir.mkdir(parents=True, exist_ok=True)

    print("### Windows SDK")
    known_win_sdk_pkg_ids_to_download = [  # noqa
        # Windows SDK tools (like rc.exe & mt.exe)
        f"Windows SDK for Windows Store Apps Tools-x86_en-us.msi",
        # Windows SDK headers
        f"Windows SDK for Windows Store Apps Headers-x86_en-us.msi",
        f"Windows SDK for Windows Store Apps Headers OnecoreUap-x86_en-us.msi",
        f"Windows SDK OnecoreUap Headers x64-x86_en-us.msi",
        f"Windows SDK OnecoreUap Headers x86-x86_en-us.msi",
        f"Windows SDK Desktop Headers x64-x86_en-us.msi",
        f"Windows SDK Desktop Headers x86-x86_en-us.msi",
        # Windows SDK libs
        f"Windows SDK for Windows Store Apps Libs-x86_en-us.msi",
        f"Windows SDK Desktop Libs x64-x86_en-us.msi",
        f"Windows SDK Desktop Libs x86-x86_en-us.msi",
        # CRT headers & libs
        f"Universal CRT Headers Libraries and Sources-x86_en-us.msi",
        # CRT redist
        # "Universal CRT Redistributable-x86_en-us.msi",
    ]

    _p1_in_mainfest = manifest[win_sdk_pkg_id_chosen_by_user][0]  # e.g. manifest[26100][0]
    _p2_in_mainfest = manifest[_first(_p1_in_mainfest["dependencies"], lambda x: True).lower()][0]

    msi_xs = []
    cabs = []

    # download msi files
    for a_pkg_id in known_win_sdk_pkg_ids_to_download:
        try:
            msiINfo_found_in_p2_paylods = _first(
                _p2_in_mainfest["payloads"], lambda x: x["fileName"] == rf"Installers\{a_pkg_id}"
            )
            msi_file_save_to = Path(sdk_save_to_dir) / "windows_sdk" / msiINfo_found_in_p2_paylods["fileName"]

            print(f'"{msi_file_save_to}"')
            print(msiINfo_found_in_p2_paylods["url"])

            msi_xs.append(msi_file_save_to)
            # find cab files that are referred in msi
            cabs += list(
                _get_msi_cabs(
                    (
                        data := _download_with_sha256(
                            msiINfo_found_in_p2_paylods["url"], msiINfo_found_in_p2_paylods["sha256"], msi_file_save_to
                        )
                    )
                )
            )
            Total_download_bytes_count += len(data)
        except StopIteration:
            print(f"cannot find {a_pkg_id}")

    # download .cab files
    for a_cab_id in cabs:
        cabInfo_found_in_p2_payloads = _first(_p2_in_mainfest["payloads"], lambda x: x["fileName"] == rf"Installers\{a_cab_id}")
        print(a_cab_id)
        print(cabInfo_found_in_p2_payloads["url"])
        cab_file_save_to = (Path() / sdk_save_to_dir / "windows_sdk" / cabInfo_found_in_p2_payloads["fileName"]).absolute()
        _download_with_sha256(cabInfo_found_in_p2_payloads["url"], cabInfo_found_in_p2_payloads["sha256"], cab_file_save_to)

    print("Unpacking msi files...")
    for m in msi_xs:
        print(m)
        print(output_dir.resolve())
        msi_extract_cmd = ["lessmsi", "x", f'"{m}"', f'"{str(output_dir)}\\"']
        _cmd_str = " ".join(msi_extract_cmd)
        subprocess.check_call(_cmd_str)  # , stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)


def download_and_place_runtimedebug(host, target, output_dir: Path, _msi_store_dir: Path, packages: dict):
    # place debug CRT runtime files into MSVC folder (not what real Visual Studio installer does... but is reasonable)
    global Total_download_bytes_count
    print("\n####", (pkg := "microsoft.visualcpp.runtimedebug.14"))

    _msi_store_dir.mkdir(parents=True, exist_ok=True)
    for payload in (_first(packages[pkg], lambda p: p["chip"] == host))["payloads"]:  # download payload files: msi and cab
        Total_download_bytes_count += len(
            _download_with_sha256(payload["url"], payload["sha256"], save_as=_msi_store_dir / payload["fileName"])
        )

    with tempfile.TemporaryDirectory() as tmp_msi_extract_dir:  # unzip msi file
        msi_extract_cmd = ["lessmsi", "x", str((next(_msi_store_dir.glob("*.msi"))).resolve()), f"{tmp_msi_extract_dir}\\"]
        subprocess.check_call(msi_extract_cmd)  # , stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

        _v = list((output_dir / "VC/Tools/MSVC").glob("*"))[0].name
        dst_dir = Path(
            output_dir / "VC/Tools/MSVC" / _v / f"bin/Host{host}/{target}"
        )  # e.g. vs2022_17.10_msvc_14.40_x64\VC\Tools\MSVC\14.40.33807\bin\Hostx64\x64
        print(f"Placing debug CRT runtime files into MSVC folder:\n{Fore.BLUE}{dst_dir}")

        for src_file in Path(tmp_msi_extract_dir).rglob("*.*"):  # src_dir.iterdir():
            if src_file.is_dir():
                continue  # noqa
            dst_file = dst_dir / src_file.name

            print(f"{Fore.GREEN}  {src_file.name.ljust(48)}", end="")

            if _calculate_sha256(src_file) == (_calculate_sha256(dst_file) if dst_file.exists() else None):
                print(f" {Fore.CYAN}OK")
            else:
                for attempt in range(10):
                    try:
                        # src_file.replace(dst_dir / src_file.name)
                        shutil.copyfile(src_file, dst_dir / src_file.name)
                        print(f" {Fore.BLUE}OK (copied, {attempt + 1} try)")
                        break
                    except Exception as e:
                        print(f"Attempt {attempt + 1} failed with error: {e}")
                        time_to_wait = random.randint(500, 1500) / 1000  # Convert milliseconds to seconds
                        print(f"Waiting for {time_to_wait} seconds before retrying...")
                        time.sleep(time_to_wait)
                else:
                    new_file_path = dst_dir / (src_file.stem + ".new" + "".join(src_file.suffixes))
                    print(" Operation failed after 10 attempts.")
                    print(f"copy {src_file} as {new_file_path}")
                    shutil.copyfile(src_file, new_file_path)
    return


def download_and_place_msdia140(host, target, output_dir: Path, _msi_store_dir: Path, packages: dict) -> None:
    # download DIA SDK and put msdia140.dll file into MSVC folder
    global Total_download_bytes_count
    print(f'\n### {(pkg := "microsoft.visualc.140.dia.sdk.msi")}')

    _msi_store_dir.mkdir(parents=True, exist_ok=True)
    for payload in packages[pkg][0]["payloads"]:
        Total_download_bytes_count += len(
            _download_with_sha256(payload["url"], payload["sha256"], save_as=_msi_store_dir / payload["fileName"])
        )

    with tempfile.TemporaryDirectory() as tmp_msi_extract_dir:  # unzip msi file
        msi_extract_cmd = ["lessmsi", "x", str((next(_msi_store_dir.glob("*.msi"))).resolve()), f"{tmp_msi_extract_dir}\\"]
        subprocess.check_call(msi_extract_cmd)  # , stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

        # temp/tmpitaryquh/SourceDir/Program Files/Microsoft Visual Studio 14.0/DIA SDK/bin/msdia140.dll
        # temp/tmpitaryquh/SourceDir/Program Files/Microsoft Visual Studio 14.0/DIA SDK/bin/amd64/msdia140.dll
        # temp/tmpitaryquh/SourceDir/Program Files/Microsoft Visual Studio 14.0/DIA SDK/bin/arm/msdia140.dll
        _msdia_dict = {
            "x86": "msdia140.dll",
            "x64": "amd64/msdia140.dll",
            "arm": "arm/msdia140.dll",
        }
        src_file = next(Path(tmp_msi_extract_dir).rglob("DIA SDK")) / "bin" / _msdia_dict[host]
        _v = list((output_dir / "VC/Tools/MSVC").glob("*"))[0].name
        dst_file = output_dir / "VC/Tools/MSVC" / _v / f"bin/Host{host}/{target}" / src_file.name

        print(f"Download DIA SDK and put msdia140.dll file into MSVC folder:\n{Fore.BLUE}{dst_file.parent}")
        if _calculate_sha256(src_file) == (_calculate_sha256(dst_file) if dst_file.exists() else None):
            print(f"{Fore.GREEN}  {_msdia_dict[host].ljust(48)}{Fore.CYAN} OK")
        else:
            for attempt in range(10):
                try:
                    shutil.copyfile(src_file, dst_file)
                    print(f"{Fore.GREEN}  {_msdia_dict[host].ljust(48)} {Fore.BLUE}OK (copied, {attempt + 1} try)")
                    break
                except Exception as e:
                    print(f"Attempt {attempt + 1} failed with error: {e}")
                    time_to_wait = random.randint(500, 1500) / 1000  # Convert milliseconds to seconds
                    print(f"Waiting for {time_to_wait} seconds before retrying...")
                    time.sleep(time_to_wait)
            else:
                new_file_path = dst_file.parent / (src_file.stem + ".new" + "".join(src_file.suffixes))
                print("Operation failed after 10 attempts.")
                print(f"copy {src_file} as {new_file_path}")
                shutil.copyfile(src_file, new_file_path)


def write_setup_bat(host: str, target: str, outdir: Path) -> None:
    msc_v = list((outdir / "VC/Tools/MSVC").glob("*"))[0].name
    sdk_v = list((outdir / "SourceDir" / "Windows Kits/10/bin").glob("*"))[0].name

    setup_bat_file_path = outdir / f"{target}_setup.bat"
    print(f"{msc_v=}, {sdk_v=}, \n{setup_bat_file_path}")

    setup_bat_file_path.write_text(
        rf"""
        @echo off
        set ROOT=%~dp0

        set MSVC_VERSION={msc_v}
        set MSVC_HOST=Host{host}
        set MSVC_ARCH={target}
        set SDK_VERSION={sdk_v}
        set SDK_ARCH={target}

        set MSVC_ROOT=%ROOT%VC\Tools\MSVC\%MSVC_VERSION%
        set SDK_INCLUDE=%ROOT%SourceDir\Windows Kits\10\Include\%SDK_VERSION%
        set SDK_LIBS=%ROOT%SourceDir\Windows Kits\10\Lib\%SDK_VERSION%

        set VCToolsInstallDir=%MSVC_ROOT%\
        set PATH=%MSVC_ROOT%\bin\%MSVC_HOST%\%MSVC_ARCH%;%ROOT%SourceDir\Windows Kits\10\bin\%SDK_VERSION%\%SDK_ARCH%;%ROOT%SourceDir\Windows Kits\10\bin\%SDK_VERSION%\%SDK_ARCH%\ucrt;%PATH%
        set INCLUDE=%MSVC_ROOT%\include;%MSVC_ROOT%\atlmfc\include;%SDK_INCLUDE%\ucrt;%SDK_INCLUDE%\shared;%SDK_INCLUDE%\um;%SDK_INCLUDE%\winrt;%SDK_INCLUDE%\cppwinrt
        set LIB=%MSVC_ROOT%\lib\%MSVC_ARCH%;%MSVC_ROOT%\atlmfc\lib\spectre\%MSVC_ARCH%;%MSVC_ROOT%\atlmfc\lib\%MSVC_ARCH%;%SDK_LIBS%\ucrt\%SDK_ARCH%;%SDK_LIBS%\um\%SDK_ARCH%

        echo VCToolsInstallDir=%VCToolsInstallDir%
        echo/
        echo INCLUDE=
        echo %MSVC_ROOT%\include
        echo %MSVC_ROOT%\atlmfc\include
        echo %SDK_INCLUDE%\ucrt
        echo %SDK_INCLUDE%\shared
        echo %SDK_INCLUDE%\um
        echo %SDK_INCLUDE%\winrt
        echo %SDK_INCLUDE%\cppwinrt
        echo/
        echo LIB=
        echo %MSVC_ROOT%\lib\%MSVC_ARCH%
        echo %MSVC_ROOT%\atlmfc\lib\spectre\%MSVC_ARCH%
        echo %MSVC_ROOT%\atlmfc\lib\%MSVC_ARCH%
        echo %SDK_LIBS%\ucrt\%SDK_ARCH%
        echo %SDK_LIBS%\um\%SDK_ARCH%
        echo/
        echo path added: 
        echo %MSVC_ROOT%\bin\%MSVC_HOST%\%MSVC_ARCH%
        echo %ROOT%SourceDir\Windows Kits\10\bin\%SDK_VERSION%\%SDK_ARCH%
        echo %ROOT%SourceDir\Windows Kits\10\bin\%SDK_VERSION%\%SDK_ARCH%\ucrt
        """
    )


def _first(items, cond):
    return next(item for item in items if cond(item))


def _calculate_sha256(file_path):
    hash_sha256 = hashlib.sha256()
    with open(file_path, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_sha256.update(chunk)
    return hash_sha256.hexdigest()


def _download(url: str, dst_file_path_: Path | None = None):
    chunk_size = 1024 * 1024
    content = io.BytesIO()

    with urllib.request.urlopen(url) as res:
        total_size = int(res.info().get("Content-Length", 0))
        with tqdm(total=total_size, unit="B", unit_scale=True, desc=f"{url}") as pbar:
            while True:
                chunk = res.read(chunk_size)
                if not chunk:
                    break
                content.write(chunk)
                pbar.update(len(chunk))

    if dst_file_path_ is not None:
        Path(dst_file_path_).write_bytes(content.getbuffer())
    return content.getvalue()


def _download_with_sha256(url: str, sha256_check: str, save_as: Path | str):
    save_as = Path(save_as)

    save_as.parent.mkdir(parents=True, exist_ok=True)

    dst_checksum = _calculate_sha256(save_as) if save_as.exists() else "None"
    if dst_checksum.lower() == sha256_check.lower():
        print(f"{Fore.GREEN}{str(save_as).ljust(50)} {Fore.CYAN}OK")

    else:
        response = requests.get(url, stream=True)

        total_size_in_bytes = int(response.headers.get("content-length", 0))
        block_size = 1024 * 1024

        sha256_hash = hashlib.sha256()
        with tqdm(total=total_size_in_bytes, unit="iB", unit_scale=True) as progress_bar, open(save_as, "wb") as file:
            for data_ in response.iter_content(block_size):
                file.write(data_)
                sha256_hash.update(data_)
                progress_bar.update(len(data_))

        if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
            print("ERROR, size wrong")
        elif sha256_hash.hexdigest().lower() != sha256_check.lower():
            print(sha256_check)
            print(f"ERROR, SHA256 mismatch: {sha256_hash.hexdigest()}")
        else:
            print("Download completed successfully, and SHA256 matched.")

    return save_as.read_bytes()


def _get_msi_cabs(msi_):  # super crappy msi format parser just to find required .cab files
    index = 0
    while True:
        index = msi_.find(b".cab", index + 4)
        if index < 0:
            return
        yield msi_[index - 32 : index + 4].decode("ascii")


def _parse_my_args():
    ap = argparse.ArgumentParser()
    ap.add_argument(
        "--show-versions",
        const=True,
        action="store_const",
        help="Show available MSVC and Windows SDK versions",
    )
    ap.add_argument("--msvc-version", help="download specific MSVC version")
    ap.add_argument("--sdk-version", help="download specific Windows SDK version")
    return ap.parse_args()


def cleanup(out_dir: Path):
    ### cleanup

    # shutil.rmtree(OUTPUT / "Common7", ignore_errors=True)

    # msvcv = list((out_dir / "VC/Tools/MSVC").glob("*"))[0].name
    # sdkv = list((out_dir / "SourceDir" / "Windows Kits/10/bin").glob("*"))[0].name
    #
    # print()
    # for f in ["Auxiliary", "lib/x64/store", "lib/x64/uwp", "lib/x86/store", "lib/x86/uwp"]:
    #     pp = out_dir / "VC/Tools/MSVC" / msvcv / f
    #     print(pp.exists(), pp)
    #     # shutil.rmtree(out_dir / "VC/Tools/MSVC" / msvc_ver / f)
    #
    # .. for f in OUTPUT.glob("*.msi"):
    # ..     f.unlink()
    #
    # print()
    # for f in ["Catalogs", "DesignTime", f"bin/{sdkv}/chpe", f"Lib/{sdkv}/ucrt_enclave"]:
    #     pp = out_dir / "SourceDir" / "Windows Kits/10" / f
    #     print(pp.exists(), pp)
    #     # shutil.rmtree(out_dir / "Windows Kits/10" / f, ignore_errors=True)
    #
    # print()
    # for arch in ["arm", "arm64"]:
    #     pp_xs = [
    #         out_dir / "SourceDir" / "Windows Kits/10/bin" / sdkv / arch,
    #         out_dir / "SourceDir" / "Windows Kits/10/Lib" / sdkv / "ucrt" / arch,
    #         out_dir / "SourceDir" / "Windows Kits/10/Lib" / sdkv / "um" / arch,
    #     ]
    #     for pp in pp_xs:
    #         print(str(pp.exists()).ljust(10), pp)

    print()
    print("AAAAAAA")
    _dir_size = lambda path: sum(ff.stat().st_size for ff in Path(path).rglob("*") if ff.is_file())
    _h = lambda size_in_bytes: next(
        f"{size_in_bytes / (1024 ** i):.2f} {unit}"
        for i, unit in enumerate(["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"])
        if size_in_bytes < 1024 ** (i + 1)
    )

    for pp in out_dir.rglob("*arm*"):
        if pp.is_dir():
            # print(_h(dir_size(pp)), pp)
            print(f"removing {pp}")
            shutil.rmtree(pp)

    # shutil.rmtree(out_dir / "VC/Tools/MSVC" / msvcv / f"bin/Host{arch}", ignore_errors=True)
    # shutil.rmtree(out_dir / "Windows Kits/10/bin" / sdkv / arch)
    # shutil.rmtree(out_dir / "Windows Kits/10/Lib" / sdkv / "ucrt" / arch)
    # shutil.rmtree(out_dir / "Windows Kits/10/Lib" / sdkv / "um" / arch)


"""
rg """ "id" """:.+Win.+SDK VisualStudio.vsman.jsonc
355143:                 "id": "Microsoft.VisualStudio.Component.Windows10SDK",
355237:                 "id": "Microsoft.VisualStudio.Component.Windows10SDK.18362",
355331:                 "id": "Microsoft.VisualStudio.Component.Windows10SDK.19041",
355425:                 "id": "Microsoft.VisualStudio.Component.Windows10SDK.20348",
355519:                 "id": "Microsoft.VisualStudio.Component.Windows10SDK.IpOverUsb",
355635:                 "id": "Microsoft.VisualStudio.Component.Windows11SDK.22000",
355729:                 "id": "Microsoft.VisualStudio.Component.Windows11SDK.22621",
355823:                 "id": "Microsoft.VisualStudio.Component.Windows11SDK.26100",
362028:                 "id": "Microsoft.VisualStudio.ComponentGroup.WindowsAppSDK.Cpp",
362210:                 "id": "Microsoft.VisualStudio.ComponentGroup.WindowsAppSDK.Cs",
479166:                 "id": "Microsoft.Windows.SDK.BuildTools_10.0.22621.3233",
479185:                 "id": "Microsoft.Windows.UniversalCRT.ExtensionSDK.Msi",
479792:                 "id": "Microsoft.WindowsAppSDK.Cpp.Dev17",
479813:                 "id": "Microsoft.WindowsAppSDK.Cs.Dev17",
486049:                 "id": "Win10SDK_10.0.18362",
488547:                 "id": "Win10SDK_10.0.19041",
491075:                 "id": "Win10SDK_10.0.20348",
493624:                 "id": "Win10SDK_IpOverUsb",
493744:                 "id": "Win11SDK_10.0.22000",
496334:                 "id": "Win11SDK_10.0.22000",
498924:                 "id": "Win11SDK_10.0.22000",
501514:                 "id": "Win11SDK_10.0.22621",
504217:                 "id": "Win11SDK_10.0.26100",
506670:                 "id": "Win11SDK_WindowsPerformanceToolkit",
"""

"""
Windows SDK Desktop Headers x86-x86_en-us.msi 
Windows SDK Desktop Libs x86-x86_en-us.msi
Windows SDK Desktop Tools x86-x86_en-us.msi
Windows SDK DirectX x86 Remote-x86_en-us.msi
Windows SDK EULA-x86_en-us.msi
Windows SDK Facade Windows WinMD Versioned-x86_en-us.msi
Windows SDK Modern Non-Versioned Developer Tools-x86_en-us.msi
Windows SDK Modern Versioned Developer Tools-x86_en-us.msi
Windows SDK OnecoreUap Headers x86-x86_en-us.msi
Windows SDK Redistributables-x86_en-us.msi
Windows SDK Signing Tools-x86_en-us.msi
Windows SDK for Windows Store Apps Contracts-x86_en-us.msi
Windows SDK for Windows Store Apps DirectX x86 Remote-x86_en-us.msi
Windows SDK for Windows Store Apps Headers OnecoreUap-x86_en-us.msi
Windows SDK for Windows Store Apps Headers-x86_en-us.msi
Windows SDK for Windows Store Apps Legacy Tools-x86_en-us.msi
Windows SDK for Windows Store Apps Libs-x86_en-us.msi
Windows SDK for Windows Store Apps Metadata-x86_en-us.msi
Windows SDK for Windows Store Apps Tools-x86_en-us.msi
Windows SDK for Windows Store Apps-x86_en-us.msi
Windows SDK for Windows Store Managed Apps Libs-x86_en-us.msi
Windows SDK-x86_en-us.msi

Windows SDK Desktop Headers x64-x86_en-us.msi
Windows SDK Desktop Libs x64-x86_en-us.msi
Windows SDK Desktop Tools x64-x86_en-us.msi
Windows SDK DirectX x64 Remote-x64_en-us.msi
Windows SDK OnecoreUap Headers x64-x86_en-us.msi
Windows SDK for Windows Store Apps DirectX x64 Remote-x64_en-us.msi
"""

# args = parse_my_args()
# _msvc_ver = args.msvc_version or max(sorted(msvc_versions.keys()))
# sdk_ver = args.sdk_version or max(sorted(sdk_versions.keys()))
# _msvc_pid = msvc_versions[_msvc_ver]


if __name__ == "__main__":
    main()