Last active
November 11, 2019 15:26
-
-
Save goldshtn/fe3f7c3b10ec7e5511ae755abaf52172 to your computer and use it in GitHub Desktop.
dotnet-mapgen: generates map files for crossgen-compiled assemblies, and merges them into the main perf map file
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # | |
| # USAGE: dotnet-mapgen [-h] {generate,merge} PID | |
| # | |
| # In generate mode, this tool reads the /tmp/perfinfo-PID.map file generated | |
| # by the CLR when running with COMPlus_PerfMapEnabled=1, and finds all load | |
| # events for managed assemblies. For each managed assembly found in this way, | |
| # the tool runs crossgen to generate a symbol mapping file (akin to debuginfo). | |
| # | |
| # In merge mode, this tool finds the load address of each managed assembly in | |
| # the target process, and merges the addresses from the crossgen-generated | |
| # map files into the main /tmp/perf-PID.map file for the process. The crossgen- | |
| # generated map files contain relative addresses, so the tool has to translate | |
| # these into absolute addresses using the load address of each managed assembly | |
| # in the target process. | |
| # | |
| # Copyright (C) 2017, Sasha Goldshtein | |
| # Licensed under the MIT License | |
| import argparse | |
| import glob | |
| import os | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| def bail(error): | |
| print("ERROR: " + error) | |
| exit(1) | |
| def get_assembly_list(pid): | |
| assemblies = [] | |
| try: | |
| with open("/tmp/perfinfo-%d.map" % pid) as f: | |
| for line in f: | |
| parts = line.split(';') | |
| if len(parts) < 2 or parts[0] != "ImageLoad": | |
| continue | |
| assemblies.append(parts[1]) | |
| except IOError: | |
| bail("error opening /tmp/perfinfo-%d.map file" % pid) | |
| return assemblies | |
| def find_libcoreclr(pid): | |
| libcoreclr = subprocess.check_output( | |
| "cat /proc/%d/maps | grep libcoreclr.so | head -1 | awk '{ print $6 }'" | |
| % pid, shell=True) | |
| return libcoreclr | |
| def download_crossgen(libcoreclr): | |
| # Updated for CoreCLR 2.0. project.json doesn't work anymore, need to generate | |
| # a .csproj and restore from that. In the meantime, portable RIDs showed up, | |
| # so we no longer need anything more specific than "linux-x64" (assuming, of | |
| # course, that we're on Linux x64). | |
| coreclr_ver = "2.0.0" | |
| rid = "linux-x64" | |
| tmp_folder = tempfile.mkdtemp(prefix="crossgen") | |
| project = os.path.join(tmp_folder, "project.csproj") | |
| with open(project, "w") as f: | |
| f.write(""" | |
| <Project Sdk="Microsoft.NET.Sdk"> | |
| <PropertyGroup> | |
| <OutputType>Exe</OutputType> | |
| <TargetFramework>netcoreapp2.0</TargetFramework> | |
| </PropertyGroup> | |
| <ItemGroup> | |
| <PackageReference Include="Microsoft.NETCore.Runtime.CoreCLR" Version="%s" /> | |
| </ItemGroup> | |
| </Project>\n""" % coreclr_ver) | |
| subprocess.check_call("dotnet restore %s --packages %s -r %s >/dev/null 2>&1" | |
| % (project, tmp_folder, rid), shell=True) | |
| shutil.copy( | |
| "%s/runtime.%s.microsoft.netcore.runtime.coreclr/%s/tools/crossgen" | |
| % (tmp_folder, rid, coreclr_ver), | |
| os.path.dirname(libcoreclr)) | |
| print("crossgen succesfully downloaded and placed in libcoreclr's dir") | |
| shutil.rmtree(tmp_folder) | |
| def find_crossgen(pid): | |
| libcoreclr = find_libcoreclr(pid) | |
| path = os.path.dirname(libcoreclr) | |
| crossgen = os.path.join(path, "crossgen") | |
| if not os.path.isfile(crossgen): | |
| print("couldn't find crossgen, trying to fetch it automatically...") | |
| download_crossgen(libcoreclr) | |
| return crossgen | |
| def generate(pid): | |
| assemblies = get_assembly_list(pid) | |
| crossgen = find_crossgen(pid) | |
| asm_list = str.join(":", assemblies) | |
| succeeded, failed = (0, 0) | |
| for assembly in assemblies: | |
| rc = subprocess.call(("%s /Trusted_Platform_Assemblies '%s' " + | |
| "/CreatePerfMap /tmp %s >/dev/null 2>&1") % | |
| (crossgen, asm_list, assembly), shell=True) | |
| if rc == 0: | |
| succeeded += 1 | |
| else: | |
| failed += 1 | |
| print("crossgen map generation: %d succeeded, %d failed" % | |
| (succeeded, failed)) | |
| def get_base_address(pid, assembly): | |
| hexaddr = subprocess.check_output( | |
| "cat /proc/%d/maps | grep %s | head -1 | cut -d '-' -f 1" % | |
| (pid, assembly), shell=True) | |
| if hexaddr == '': | |
| return -1 | |
| return int(hexaddr, 16) | |
| def append_perf_map(assembly, asm_map, pid): | |
| base_address = get_base_address(pid, assembly) | |
| lines_to_add = "" | |
| with open(asm_map) as f: | |
| for line in f: | |
| parts = line.split() | |
| offset, size, symbol = parts[0], parts[1], str.join(" ", parts[2:]) | |
| offset = int(offset, 16) + base_address | |
| lines_to_add += "%016x %s %s\n" % (offset, size, symbol) | |
| with open("/tmp/perf-%d.map" % pid, "a") as perfmap: | |
| perfmap.write(lines_to_add) | |
| def merge(pid): | |
| assemblies = get_assembly_list(pid) | |
| succeeded, failed = (0, 0) | |
| for assembly in assemblies: | |
| # TODO The generated map files have a GUID embedded in them, which | |
| # allows multiple versions to coexist (probably). How do we get | |
| # this GUID? E.g.: | |
| # System.Runtime.ni.{819d412e-d773-4dbb-8d01-20d412b6cf09}.map | |
| matches = glob.glob("/tmp/%s.ni.{*}.map" % | |
| os.path.splitext(os.path.basename(assembly))[0]) | |
| if len(matches) == 0: | |
| failed += 1 | |
| else: | |
| append_perf_map(assembly, matches[0], pid) | |
| succeeded += 1 | |
| print("perfmap merging: %d succeeded, %d failed" % (succeeded, failed)) | |
| parser = argparse.ArgumentParser(description= | |
| "Generates map files for crossgen-compiled assemblies, and merges them " + | |
| "into the main perf map file. Built for use with .NET Core on Linux.") | |
| parser.add_argument("action", choices=["generate", "merge"], | |
| help="the action to perform") | |
| parser.add_argument("pid", type=int, help="the dotnet process id") | |
| args = parser.parse_args() | |
| if args.action == "generate": | |
| generate(args.pid) | |
| elif args.action == "merge": | |
| merge(args.pid) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # | |
| # USAGE: dotnet-mapgen [-h] {generate,merge} PID | |
| # | |
| # In generate mode, this tool reads the /tmp/perfinfo-PID.map file generated | |
| # by the CLR when running with COMPlus_PerfMapEnabled=1, and finds all load | |
| # events for managed assemblies. For each managed assembly found in this way, | |
| # the tool runs crossgen to generate a symbol mapping file (akin to debuginfo). | |
| # | |
| # In merge mode, this tool finds the load address of each managed assembly in | |
| # the target process, and merges the addresses from the crossgen-generated | |
| # map files into the main /tmp/perf-PID.map file for the process. The crossgen- | |
| # generated map files contain relative addresses, so the tool has to translate | |
| # these into absolute addresses using the load address of each managed assembly | |
| # in the target process. | |
| # | |
| # Copyright (C) 2018, Sasha Goldshtein | |
| # Licensed under the MIT License | |
| import argparse | |
| import glob | |
| import os | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| def bail(error): | |
| print("ERROR: " + error) | |
| exit(1) | |
| def get_assembly_list(pid): | |
| assemblies = [] | |
| try: | |
| with open("/tmp/perfinfo-%d.map" % pid) as f: | |
| for line in f: | |
| parts = line.split(';') | |
| if len(parts) < 2 or parts[0] != "ImageLoad": | |
| continue | |
| assemblies.append(parts[1]) | |
| except IOError: | |
| bail("error opening /tmp/perfinfo-%d.map file" % pid) | |
| return assemblies | |
| def find_libcoreclr(pid): | |
| libcoreclr = subprocess.check_output( | |
| "cat /proc/%d/maps | grep libcoreclr.so | head -1 | awk '{ print $6 }'" | |
| % pid, shell=True) | |
| return libcoreclr | |
| def download_crossgen(libcoreclr): | |
| # Updated for CoreCLR 2.0. project.json doesn't work anymore, need to generate | |
| # a .csproj and restore from that. In the meantime, portable RIDs showed up, | |
| # so we no longer need anything more specific than "linux-x64" (assuming, of | |
| # course, that we're on Linux x64). | |
| coreclr_ver = "2.0.0" | |
| rid = "linux-x64" | |
| tmp_folder = tempfile.mkdtemp(prefix="crossgen") | |
| project = os.path.join(tmp_folder, "project.csproj") | |
| with open(project, "w") as f: | |
| f.write(""" | |
| <Project Sdk="Microsoft.NET.Sdk"> | |
| <PropertyGroup> | |
| <OutputType>Exe</OutputType> | |
| <TargetFramework>netcoreapp2.0</TargetFramework> | |
| </PropertyGroup> | |
| <ItemGroup> | |
| <PackageReference Include="Microsoft.NETCore.Runtime.CoreCLR" Version="%s" /> | |
| </ItemGroup> | |
| </Project>\n""" % coreclr_ver) | |
| subprocess.check_call(("sudo docker run --rm -it -v %s:/app microsoft/dotnet:sdk " + | |
| "-u=$UID:$(id -g $USER)" + | |
| "bash -c \"dotnet restore /app/project.csproj --packages /app " + | |
| "-r %s >/dev/null 2>&1\"") | |
| % (tmp_folder, rid), shell=True) | |
| shutil.copy( | |
| "%s/runtime.%s.microsoft.netcore.runtime.coreclr/%s/tools/crossgen" | |
| % (tmp_folder, rid, coreclr_ver), | |
| os.path.dirname(libcoreclr)) | |
| print("crossgen succesfully downloaded and placed in libcoreclr's dir") | |
| shutil.rmtree(tmp_folder) | |
| def find_crossgen(pid): | |
| libcoreclr = find_libcoreclr(pid) | |
| path = os.path.dirname(libcoreclr) | |
| crossgen = os.path.join(path, "crossgen") | |
| if not os.path.isfile(crossgen): | |
| print("couldn't find crossgen, trying to fetch it automatically...") | |
| download_crossgen(libcoreclr) | |
| return crossgen | |
| def generate(pid): | |
| assemblies = get_assembly_list(pid) | |
| crossgen = find_crossgen(pid) | |
| asm_list = str.join(":", assemblies) | |
| succeeded, failed = (0, 0) | |
| for assembly in assemblies: | |
| rc = subprocess.call(("%s /Trusted_Platform_Assemblies '%s' " + | |
| "/CreatePerfMap /tmp %s >/dev/null 2>&1") % | |
| (crossgen, asm_list, assembly), shell=True) | |
| if rc == 0: | |
| succeeded += 1 | |
| else: | |
| failed += 1 | |
| print("crossgen map generation: %d succeeded, %d failed" % | |
| (succeeded, failed)) | |
| def get_base_address(pid, assembly): | |
| hexaddr = subprocess.check_output( | |
| "cat /proc/%d/maps | grep %s | head -1 | cut -d '-' -f 1" % | |
| (pid, assembly), shell=True) | |
| if hexaddr == '': | |
| return -1 | |
| return int(hexaddr, 16) | |
| def append_perf_map(assembly, asm_map, pid): | |
| base_address = get_base_address(pid, assembly) | |
| lines_to_add = "" | |
| with open(asm_map) as f: | |
| for line in f: | |
| parts = line.split() | |
| offset, size, symbol = parts[0], parts[1], str.join(" ", parts[2:]) | |
| offset = int(offset, 16) + base_address | |
| lines_to_add += "%016x %s %s\n" % (offset, size, symbol) | |
| with open("/tmp/perf-%d.map" % pid, "a") as perfmap: | |
| perfmap.write(lines_to_add) | |
| def merge(pid): | |
| assemblies = get_assembly_list(pid) | |
| succeeded, failed = (0, 0) | |
| for assembly in assemblies: | |
| # TODO The generated map files have a GUID embedded in them, which | |
| # allows multiple versions to coexist (probably). How do we get | |
| # this GUID? E.g.: | |
| # System.Runtime.ni.{819d412e-d773-4dbb-8d01-20d412b6cf09}.map | |
| matches = glob.glob("/tmp/%s.ni.{*}.map" % | |
| os.path.splitext(os.path.basename(assembly))[0]) | |
| if len(matches) == 0: | |
| failed += 1 | |
| else: | |
| append_perf_map(assembly, matches[0], pid) | |
| succeeded += 1 | |
| print("perfmap merging: %d succeeded, %d failed" % (succeeded, failed)) | |
| parser = argparse.ArgumentParser(description= | |
| "Generates map files for crossgen-compiled assemblies, and merges them " + | |
| "into the main perf map file. Built for use with .NET Core on Linux.") | |
| parser.add_argument("action", choices=["generate", "merge"], | |
| help="the action to perform") | |
| parser.add_argument("pid", type=int, help="the dotnet process id") | |
| args = parser.parse_args() | |
| if args.action == "generate": | |
| generate(args.pid) | |
| elif args.action == "merge": | |
| merge(args.pid) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # | |
| # USAGE: dotnet-mapgen [-h] {generate,merge} PID | |
| # | |
| # In generate mode, this tool reads the /tmp/perfinfo-PID.map file generated | |
| # by the CLR when running with COMPlus_PerfMapEnabled=1, and finds all load | |
| # events for managed assemblies. For each managed assembly found in this way, | |
| # the tool runs crossgen to generate a symbol mapping file (akin to debuginfo). | |
| # | |
| # In merge mode, this tool finds the load address of each managed assembly in | |
| # the target process, and merges the addresses from the crossgen-generated | |
| # map files into the main /tmp/perf-PID.map file for the process. The crossgen- | |
| # generated map files contain relative addresses, so the tool has to translate | |
| # these into absolute addresses using the load address of each managed assembly | |
| # in the target process. | |
| # | |
| # Copyright (C) 2017, Sasha Goldshtein | |
| # Licensed under the MIT License | |
| import argparse | |
| import glob | |
| import os | |
| import shutil | |
| import subprocess | |
| import tempfile | |
| def bail(error): | |
| print("ERROR: " + error) | |
| exit(1) | |
| def get_assembly_list(pid): | |
| assemblies = [] | |
| try: | |
| with open("/tmp/perfinfo-%d.map" % pid) as f: | |
| for line in f: | |
| parts = line.split(';') | |
| if len(parts) < 2 or parts[0] != "ImageLoad": | |
| continue | |
| assemblies.append(parts[1]) | |
| except IOError: | |
| bail("error opening /tmp/perfinfo-%d.map file" % pid) | |
| return assemblies | |
| def find_libcoreclr(pid): | |
| libcoreclr = subprocess.check_output( | |
| "cat /proc/%d/maps | grep libcoreclr.so | head -1 | awk '{ print $6 }'" | |
| % pid, shell=True) | |
| return libcoreclr | |
| def download_crossgen(libcoreclr): | |
| coreclr_ver = os.path.basename(os.path.dirname(libcoreclr)) | |
| rid = subprocess.check_output( | |
| "dotnet --info | grep RID", shell=True).split()[1] | |
| tmp_folder = tempfile.mkdtemp(prefix="crossgen") | |
| project_json = os.path.join(tmp_folder, "project.json") | |
| with open(project_json, "w") as f: | |
| f.write( | |
| """ | |
| { | |
| "frameworks": { | |
| "netcoreapp1.1": { | |
| "dependencies": { | |
| "Microsoft.NETCore.Runtime.CoreCLR": "%s", | |
| "Microsoft.NETCore.Platforms": "%s" | |
| } | |
| } | |
| }, | |
| "runtimes": { | |
| "%s": {} | |
| } | |
| }\n""" | |
| % (coreclr_ver, coreclr_ver, rid)) | |
| subprocess.check_call("dotnet restore %s --packages %s >/dev/null 2>&1" | |
| % (project_json, tmp_folder), shell=True) | |
| shutil.copy( | |
| "%s/runtime.%s.Microsoft.NETCore.Runtime.CoreCLR/%s/tools/crossgen" | |
| % (tmp_folder, rid, coreclr_ver), | |
| os.path.dirname(libcoreclr)) | |
| print("crossgen succesfully downloaded and placed in libcoreclr's dir") | |
| shutil.rmtree(tmp_folder) | |
| def find_crossgen(pid): | |
| libcoreclr = find_libcoreclr(pid) | |
| path = os.path.dirname(libcoreclr) | |
| crossgen = os.path.join(path, "crossgen") | |
| if not os.path.isfile(crossgen): | |
| print("couldn't find crossgen, trying to fetch it automatically...") | |
| download_crossgen(libcoreclr) | |
| return crossgen | |
| def generate(pid): | |
| assemblies = get_assembly_list(pid) | |
| crossgen = find_crossgen(pid) | |
| asm_list = str.join(":", assemblies) | |
| succeeded, failed = (0, 0) | |
| for assembly in assemblies: | |
| rc = subprocess.call(("%s /Trusted_Platform_Assemblies '%s' " + | |
| "/CreatePerfMap /tmp %s >/dev/null 2>&1") % | |
| (crossgen, asm_list, assembly), shell=True) | |
| if rc == 0: | |
| succeeded += 1 | |
| else: | |
| failed += 1 | |
| print("crossgen map generation: %d succeeded, %d failed" % | |
| (succeeded, failed)) | |
| def get_base_address(pid, assembly): | |
| hexaddr = subprocess.check_output( | |
| "cat /proc/%d/maps | grep %s | head -1 | cut -d '-' -f 1" % | |
| (pid, assembly), shell=True) | |
| if hexaddr == '': | |
| return -1 | |
| return int(hexaddr, 16) | |
| def append_perf_map(assembly, asm_map, pid): | |
| base_address = get_base_address(pid, assembly) | |
| lines_to_add = "" | |
| with open(asm_map) as f: | |
| for line in f: | |
| parts = line.split() | |
| offset, size, symbol = parts[0], parts[1], str.join(" ", parts[2:]) | |
| offset = int(offset, 16) + base_address | |
| lines_to_add += "%016x %s %s\n" % (offset, size, symbol) | |
| with open("/tmp/perf-%d.map" % pid, "a") as perfmap: | |
| perfmap.write(lines_to_add) | |
| def merge(pid): | |
| assemblies = get_assembly_list(pid) | |
| succeeded, failed = (0, 0) | |
| for assembly in assemblies: | |
| # TODO The generated map files have a GUID embedded in them, which | |
| # allows multiple versions to coexist (probably). How do we get | |
| # this GUID? E.g.: | |
| # System.Runtime.ni.{819d412e-d773-4dbb-8d01-20d412b6cf09}.map | |
| matches = glob.glob("/tmp/%s.ni.{*}.map" % | |
| os.path.splitext(os.path.basename(assembly))[0]) | |
| if len(matches) == 0: | |
| failed += 1 | |
| else: | |
| append_perf_map(assembly, matches[0], pid) | |
| succeeded += 1 | |
| print("perfmap merging: %d succeeded, %d failed" % (succeeded, failed)) | |
| parser = argparse.ArgumentParser(description= | |
| "Generates map files for crossgen-compiled assemblies, and merges them " + | |
| "into the main perf map file. Built for use with .NET Core on Linux.") | |
| parser.add_argument("action", choices=["generate", "merge"], | |
| help="the action to perform") | |
| parser.add_argument("pid", type=int, help="the dotnet process id") | |
| args = parser.parse_args() | |
| if args.action == "generate": | |
| generate(args.pid) | |
| elif args.action == "merge": | |
| merge(args.pid) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
v3 does not require a .NET Core installation on the host, but it uses Docker.