Debugging an external process through Microsoft Visual Studio Unit Test

An update to this article while the code is useful I did not end up using this because I was unable to control the process from within the unit test. Visual studio operates each unit test in an isolated app domain which makes it difficult to instantiate processes across the test run cycle. See my next post on how I have solved this particularity annoying problem. I have been met with an obstacle nearly every step of the way with the Visual Studio Unit tests. Had I not been switching to Teams I would abandon the whole thing and go back to NUnit which was a much more flexible framework.

Debugging a client server type model within Microsoft Visual Studio can be challenging when also trying to leverage the Microsoft Visual Studio Unit Test framework. The problem lies in the fact you cannot easily spin of a second process to host the server side communication and debug both processes at the same time. You will find that the debugger will only debug through the unit test or client side of the communication stack. After some research and testing I have found a way to launch and attach to a process within the unit test execution path without any extra foot work.

The following code will launch a process check to see if the code is within a debugging session then attempt to attach to that process. If you have been googleing you may have found that you can access the visual studio environment using:

(EnvDTE._DTE)System.Runtime.InteropServices.Marshal.GetActiveObject("VisualStudio.DTE")

This is not the droid you are looking for, While this returns an instance of visual studio it may not be the executing instance you need to attach the debugging session to.

Thanks to Rick Strahl for publishing an article with the right COM+ incantations to retrieve all active instances the DTE I made some modifications for my purposes and viola! Something that seems like it should be so simple turns out to be quite tricky. The message filter was taken directly from the MSDN site to resolve issues with RPC_E_CALL_REJECTED message. I also as a side note I was never able to debug throgh the property that returns the executing environment. This was due to re-entry, I was in a debug session at a break point while the other process was trying to attach to the current environment context which was returning as busy which is why you will get the RPC_E_CALL_REJECTED during debugging.

 public sealed class ExecuteAndAttachToBehaviorAttribute : TestFixtureBehaviorAttribute
 {
  private static EnvDTE.DTE __executingDevelopmentToolsEnvironment;
  private const uint S_OK = 0;

  public ExecuteAndAttachToBehaviorAttribute()
  {
   WaitForProcessInMilliseconds = 0;
  }

  #region Properties

  /// 

  /// Path to the executable relative to the currently executing ApplicationBase path
  /// 
  public string Path { get; set; }

  public int WaitForProcessInMilliseconds { get; set; }

  private Process Process { get; set; }
  
  private EnvDTE.DTE ExecutingDevelopmentToolsEnvironment
  {
   get
   {
    UCOMIRunningObjectTable rot;
    if (__executingDevelopmentToolsEnvironment == null && GetRunningObjectTable(0, out rot) == S_OK)
    {
     UCOMIEnumMoniker enumMon;
     rot.EnumRunning(out enumMon);
     if (enumMon != null)
     {
      const int numMons = 100;
      int Fetched = 0;
      UCOMIMoniker[] aMons = new UCOMIMoniker[numMons];
      enumMon.Next(numMons, aMons, out Fetched);
      UCOMIBindCtx ctx;

      if (CreateBindCtx(0, out ctx) == S_OK)
      {
       MessageFilter.Register();
       System.Diagnostics.Process currentProcess = System.Diagnostics.Process.GetCurrentProcess();
       for (int i = 0; i < Fetched; i++)
       {
        object instance;
        EnvDTE.DTE dte;
        rot.GetObject(aMons[i], out instance);
        dte = instance as EnvDTE.DTE;
        if (dte != null)
        {
         foreach (EnvDTE.Process p in dte.Debugger.DebuggedProcesses)
         {
          if (p.ProcessID == currentProcess.Id)
          {
           __executingDevelopmentToolsEnvironment = dte;
           break;
          }
         }
         if (__executingDevelopmentToolsEnvironment != null)
         {
          break;
         }
        }
       }
       MessageFilter.Unregister();
      }
     }
    }
    return __executingDevelopmentToolsEnvironment;
   }
  }

  #endregion

  #region Methods

  [DllImport("ole32.dll", EntryPoint = "GetRunningObjectTable")]
  private static extern uint GetRunningObjectTable(uint res, out UCOMIRunningObjectTable ROT);

  [DllImport("ole32.dll", EntryPoint = "CreateBindCtx")]
  private static extern uint CreateBindCtx(uint res, out UCOMIBindCtx ctx);

  private void Process_ErrorDataReceived(object sender, DataReceivedEventArgs e)
  {
   System.Diagnostics.Debug.WriteLine(String.Format("Unable to launch {0} the following error information was returned from the process:\r\n{1}", Path, e.Data));
  }

  public override void Setup()
  {
   Process = new Process();
   Process.StartInfo = new ProcessStartInfo(Path);
   Process.Start();
   Process.ErrorDataReceived += new DataReceivedEventHandler(Process_ErrorDataReceived);
#if DEBUG
   if (System.Diagnostics.Debugger.IsAttached)
   {
    try
    {
     if (WaitForProcessInMilliseconds > 0)
     {
      Thread.Sleep(WaitForProcessInMilliseconds);
     }
     if (ExecutingDevelopmentToolsEnvironment == null)
      throw new Exception("Unable to locate a valid instance of Visual Studio 2008, unable to attach using the current debugger instance.");
     bool attached = false;
     foreach (EnvDTE.Process process in ExecutingDevelopmentToolsEnvironment.Debugger.LocalProcesses)
     {
      if (process.ProcessID == Process.Id)
      {
       process.Attach();
       attached = true;
      }
     }
     if (!attached)
      throw new Exception(String.Format("Unable to locate process id {0}, attach failed to complete successfully", Process.Id));
    }
    catch (Exception exception)
    {
     System.Diagnostics.Debug.WriteLine(String.Format("Unable to attach debugger to '{0}' the following exception was generated:\r\n{1}", Path, exception.ToString()));
     System.Diagnostics.Debugger.Launch();

    }
   }
#endif
  }

  public override void TearDown()
  {
   try
   {
    Process.Kill();
    Process.Dispose();
   }
   catch (Exception exception)
   {
    System.Diagnostics.Debug.WriteLine(String.Format("Unable to kill {0} the following error information was returned:\r\n{1}", Path, exception.ToString()));
   }
  }

  #endregion

  private class MessageFilter : IOleMessageFilter
  {
   public static void Register()
   {
    IOleMessageFilter newFilter = new MessageFilter();
    IOleMessageFilter oldFilter = null;
    CoRegisterMessageFilter(newFilter, out oldFilter);
   }

   public static void Unregister()
   {
    IOleMessageFilter oldFilter = null;
    CoRegisterMessageFilter(null, out oldFilter);
   }

   int IOleMessageFilter.HandleInComingCall(int dwCallType, System.IntPtr hTaskCaller, int dwTickCount, System.IntPtr lpInterfaceInfo)
   {
    return 0;
   }

   int IOleMessageFilter.RetryRejectedCall(System.IntPtr hTaskCallee, int dwTickCount, int dwRejectType)
   {
    if (dwRejectType == 2)
    {
     return 99;
    }
    return -1;
   }

   int IOleMessageFilter.MessagePending(System.IntPtr hTaskCallee,int dwTickCount, int dwPendingType)
   {
    return 2;
   }

   [DllImport("Ole32.dll")]
   private static extern int
     CoRegisterMessageFilter(IOleMessageFilter newFilter, out IOleMessageFilter oldFilter);
  }

  [ComImport(), Guid("00000016-0000-0000-C000-000000000046"),
  InterfaceTypeAttribute(ComInterfaceType.InterfaceIsIUnknown)]
  interface IOleMessageFilter
  {
   [PreserveSig]
   int HandleInComingCall(
    int dwCallType,
    IntPtr hTaskCaller,
    int dwTickCount,
    IntPtr lpInterfaceInfo);

   [PreserveSig]
   int RetryRejectedCall(
    IntPtr hTaskCallee,
    int dwTickCount,
    int dwRejectType);

   [PreserveSig]
   int MessagePending(
    IntPtr hTaskCallee,
    int dwTickCount,
    int dwPendingType);
  }
 }