AttachToProcess.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.ComTypes;
using System.Threading;
using System.Threading.Tasks;
using EnvDTE80;
using EnvDTE90a;

namespace Common {
    public static class Debugging {
        private static DTE2 Dte;
        private static readonly object DteLock = new object();
        private static bool Initialized;

        public static int? GetCurrentDebuggerPid() {
                var taskCompletion = new TaskCompletionSource<int?>();
                var staThread = new System.Threading.Thread(() => {
                    try {
                        int? debuggerPid = null;
                        using (new MessageFilter()) {
                            using (var currentProcess = System.Diagnostics.Process.GetCurrentProcess())
                            using (var vsInstances = System.Diagnostics.Process.GetProcessesByName("devenv").AsDisposable()) {
                                foreach (var p in vsInstances.Enumerable) {
                                    DTE2 dte;
                                    //Will return false if target process doesn't have the same elevated rights as current process.
                                    if (TryGetVSInstance(p.Id, out dte)) {
                                        Utils.Retry(() => {
                                            var debugger = dte.Debugger; //May hang if a modal dialog is opened in Visual Studio, for example asking to reload the project.
                                            if (debugger != null) {
                                                foreach (Process2 process in debugger.DebuggedProcesses) {
                                                    if (process.ProcessID == currentProcess.Id) {
                                                        debuggerPid = p.Id;
                                                        break;
                                                    }
                                                }
                                            }
                                            Marshal.ReleaseComObject(dte);
                                        }, nbRetries: int.MaxValue, msInterval: 1000, retryOnlyOnExceptionTypes: typeof(COMException).InArray());
                                        if (debuggerPid != null) break;
                                    }
                                }
                            }
                        }
                        taskCompletion.SetResult(debuggerPid);
                    }
                    catch (Exception ex) {
                        taskCompletion.TrySetException(ex);
                    }
                }) { IsBackground = true };
                staThread.SetApartmentState(ApartmentState.STA);
                staThread.Start();
                taskCompletion.Task.Wait();
                return taskCompletion.Task.Result;
        }

        public static void AttachCurrentProcessToDebugger(int debuggerPid) {
                var taskCompletion = new TaskCompletionSource<bool>();
                var staThread = new System.Threading.Thread(() => {
                    try {
                        using (new MessageFilter()) {
                            using (var currentProcess = System.Diagnostics.Process.GetCurrentProcess()) {
                                DTE2 dte;
                                //Will return false if target process doesn't have the same elevated rights as current process.
                                if (TryGetVSInstance(debuggerPid, out dte)) {
                                    Utils.Retry(() => {
                                        var debugger = dte.Debugger; //May hang if a modal dialog is opened in Visual Studio, for example asking to reload the project.
                                        if (debugger != null) {
                                            foreach (Process4 process in debugger.LocalProcesses) {
                                                if (process.ProcessID == currentProcess.Id) {
                                                    process.Attach2("Managed"); //Managed/Native
                                                                                //debugger.CurrentProcess = process;
                                                }
                                            }
                                        }
                                        Marshal.ReleaseComObject(dte);
                                    }, nbRetries: int.MaxValue, msInterval: 1000, retryOnlyOnExceptionTypes: typeof(COMException).InArray());
                                }
                            }
                        }
                        taskCompletion.SetResult(false);
                    }
                    catch (Exception ex) {
                        taskCompletion.TrySetException(ex);
                    }
                }) { IsBackground = true };
                staThread.SetApartmentState(ApartmentState.STA);
                staThread.Start();
                taskCompletion.Task.Wait();
        }

        public static void AttachCurrentDebuggerToProcess(int processId) {
                lock (DteLock) {
                    var taskCompletion = new TaskCompletionSource<bool>();
                    var staThread = new System.Threading.Thread(() => {
                        try {
                            // Register the IOleMessageFilter to handle any threading errors.
                            using (new MessageFilter()) {
                                if (!Initialized) {
                                    using (var currentProcess = System.Diagnostics.Process.GetCurrentProcess())
                                    using (var vsInstances = System.Diagnostics.Process.GetProcessesByName("devenv").AsDisposable()) {
                                        foreach (var p in vsInstances.Enumerable) {
                                            DTE2 dte;
                                            //Will return false if target process doesn't have the same elevated rights as current process.
                                            if (TryGetVSInstance(p.Id, out dte)) {
                                                Utils.Retry(() => {
                                                    var debugger = dte.Debugger; //May hang if a modal dialog is opened in Visual Studio, for example asking to reload the project.
                                                    if (debugger != null) {
                                                        foreach (Process2 process in debugger.DebuggedProcesses) {
                                                            if (process.ProcessID == currentProcess.Id) {
                                                                Dte = dte;
                                                                break;
                                                            }
                                                        }
                                                    }
                                                }, nbRetries: int.MaxValue, msInterval: 1000, retryOnlyOnExceptionTypes: typeof(COMException).InArray());
                                                if (Dte != null) break;
                                            }
                                        }
                                    }
                                    Initialized = true;
                                    taskCompletion.SetResult(false);
                                }
                                if (Dte != null) {
                                    foreach (Process2 process in Dte.Debugger.LocalProcesses) {
                                        if (process.ProcessID == processId) {
                                            process.Attach2("Managed"); //Managed/Native
                                            //Dte.Debugger.CurrentProcess = process;
                                        }
                                    }
                                }
                            }
                            taskCompletion.SetResult(false);
                        }
                        catch (Exception ex) {
                            taskCompletion.TrySetException(ex);
                        }
                    });
                    staThread.SetApartmentState(ApartmentState.STA);
                    staThread.Start();
                    taskCompletion.Task.Wait();
                }
        }
        
        //From http://blogs.msdn.com/b/kirillosenkov/archive/2011/08/10/how-to-get-dte-from-visual-studio-process-id.aspx
        public static bool TryGetVSInstance(int processId, out DTE2 instance) {
            IBindCtx bindCtx = null;
            IRunningObjectTable rot = null;
            IEnumMoniker enumMonikers = null;

            try {
                Marshal.ThrowExceptionForHR(CreateBindCtx(reserved: 0, ppbc: out bindCtx));
                bindCtx.GetRunningObjectTable(out rot);
                rot.EnumRunning(out enumMonikers);

                IMoniker[] moniker = new IMoniker[1];
                IntPtr numberFetched = IntPtr.Zero;
                while (enumMonikers.Next(1, moniker, numberFetched) == 0) {
                    IMoniker runningObjectMoniker = moniker[0];

                    string name = null;

                    try {
                        if (runningObjectMoniker != null) {
                            runningObjectMoniker.GetDisplayName(bindCtx, null, out name);
                        }
                    }
                    catch (UnauthorizedAccessException) {
                        // Do nothing, there is something in the ROT that we do not have access to.
                    }

                    if (!string.IsNullOrEmpty(name) && name.StartsWith("!VisualStudio")) {
                        var cols = name.Split(':');
                        if (cols.Length >= 2) {
                            var currentProcessId = int.Parse(cols[1]);
                            if (currentProcessId == processId) {
                                object runningObject;
                                Marshal.ThrowExceptionForHR(rot.GetObject(runningObjectMoniker, out runningObject));
                                var dte2 = runningObject as DTE2;
                                if (dte2 != null) {
                                    instance = dte2;
                                    return true;
                                }
                            }
                        }
                    }
                }
                instance = null;
                return false;
            }
            finally {
                //http://blogs.msdn.com/b/visualstudio/archive/2010/03/01/marshal-releasecomobject-considered-dangerous.aspx
                if (enumMonikers != null) {
                    Marshal.ReleaseComObject(enumMonikers);
                }
                if (rot != null) {
                    Marshal.ReleaseComObject(rot);
                }
                if (bindCtx != null) {
                    Marshal.ReleaseComObject(bindCtx);
                }
            }
        }

        //See also https://msdn.microsoft.com/en-us/library/ms228772.aspx?f=255&MSPPError=-2147217396
        public class MessageFilter : MarshalByRefObject, IDisposable, IOleMessageFilter {
            [DllImport("ole32.dll")]
            [PreserveSig]
            private static extern int CoRegisterMessageFilter(IOleMessageFilter lpMessageFilter, out IOleMessageFilter lplpMessageFilter);

            private readonly IOleMessageFilter oldFilter;
            private const int SERVERCALL_ISHANDLED = 0;
            private const int PENDINGMSG_WAITNOPROCESS = 2;
            private const int SERVERCALL_RETRYLATER = 2;

            public MessageFilter() {
                //Starting IOleMessageFilter for COM objects
                int hr = CoRegisterMessageFilter(this, out this.oldFilter);
                System.Diagnostics.Debug.Assert(hr >= 0, "Registering COM IOleMessageFilter failed!");
            }

            public void Dispose() {
                //disabling IOleMessageFilter
                IOleMessageFilter dummy;
                int hr = CoRegisterMessageFilter(this.oldFilter, out dummy);
                System.Diagnostics.Debug.Assert(hr >= 0, "De-Registering COM IOleMessageFilter failed!");
                GC.SuppressFinalize(this);
            }

            int IOleMessageFilter.HandleInComingCall(int dwCallType, IntPtr threadIdCaller, int dwTickCount, IntPtr lpInterfaceInfo) {
                // Return the ole default (don't let the call through).
                return SERVERCALL_ISHANDLED;
            }

            int IOleMessageFilter.RetryRejectedCall(IntPtr threadIDCallee, int dwTickCount, int dwRejectType) {
                if (dwRejectType == SERVERCALL_RETRYLATER) {
                    // Retry the thread call immediately if return >=0 & 
                    // <100.
                    return 150; //waiting 150 mseconds until retry
                }
                // Too busy; cancel call. SERVERCALL_REJECTED
                return -1;
                //Call was rejected by callee. 
                //(Exception from HRESULT: 0x80010001 (RPC_E_CALL_REJECTED))
            }

            int IOleMessageFilter.MessagePending(IntPtr threadIDCallee, int dwTickCount, int dwPendingType) {
                // Perform default processing.
                return PENDINGMSG_WAITNOPROCESS;
            }
        }

        [ComImport, Guid("00000016-0000-0000-C000-000000000046"),
        InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
        interface IOleMessageFilter {
            [PreserveSig]
            int HandleInComingCall(int dwCallType, IntPtr hTaskCaller, int dwTickCount, IntPtr lpInterfaceInfo);

            [PreserveSig]
            int RetryRejectedCall(IntPtr hTaskCallee, int dwTickCount, int dwRejectType);

            [PreserveSig]
            int MessagePending(IntPtr hTaskCallee, int dwTickCount, int dwPendingType);
        }

        [DllImport("ole32.dll")]
        private static extern int CreateBindCtx(int reserved, out IBindCtx ppbc);

        [DllImport("ole32.dll")]
        private static extern int GetRunningObjectTable(int reserved, out IRunningObjectTable prot);
    }

    public static class CollectionsExtensions {
        public static T[] InArray<T>(this T item) {
            return new[] { item };
        }

        public static DisposableEnumerable<T> AsDisposable<T>(this IEnumerable<T> enumerable) where T : IDisposable {
            return new DisposableEnumerable<T>(enumerable);
        }
    }

    public class DisposableEnumerable<T> : IDisposable where T : IDisposable {
        public IEnumerable<T> Enumerable { get; }

        public DisposableEnumerable(IEnumerable<T> enumerable) {
            this.Enumerable = enumerable;
        }

        public void Dispose() {
            foreach (var o in this.Enumerable) o.Dispose();
        }
    }

    public static class Utils {
        public static void Retry(Action action, int nbRetries, int msInterval, IEnumerable<Type> retryOnlyOnExceptionTypes = null) {
            if (action == null) throw new ArgumentNullException(nameof(action));
            if (msInterval < 0) throw new ArgumentOutOfRangeException(nameof(msInterval), "msInterval must be >= 0.");
            do {
                try {
                    action();
                    return;
                }
                catch (Exception ex) {
                    if (nbRetries <= 0) throw;
                    if ((retryOnlyOnExceptionTypes != null) && !retryOnlyOnExceptionTypes.Any(e => e.IsInstanceOfType(ex))) throw;
                    Thread.Sleep(msInterval);
                }
            } while (nbRetries-- > 0);
        }
    }
}