Keeping calling process running

Aug 18, 2009 at 4:25 PM

I've been working on applications which should hook the getopenfilename and getsavefilename dialogs and replace them with the ones of my own. Everything else is working nicely but the application that launches the hook becomes unresponsive while execution is staying in the hook function. Is there any way to avoid this or am I screwed?

namespace DialogHookInject
{
    public class Main : EasyHook.IEntryPoint
    {
        DialogHook.DialogHookInterface Interface;
        LocalHook OpenFileHook;
        LocalHook SaveFileAsHook;

        public Main(RemoteHooking.IContext InContext, String InChannelName)
        {
            // connect to host...
            Interface = RemoteHooking.IpcConnectClient<DialogHook.DialogHookInterface>(InChannelName);
            Interface.Ping();
        }

        public void Run(RemoteHooking.IContext InContext, String InChannelName)
        {
            // install hooks...
            try
            {
                OpenFileHook = LocalHook.Create(
                    LocalHook.GetProcAddress("comdlg32.dll", "GetOpenFileNameW"),
                    new DGetOpenFileName(GetOpenFileName_Hooked),
                    this);

                OpenFileHook.ThreadACL.SetExclusiveACL(new Int32[] { 0 });

                SaveFileAsHook = LocalHook.Create(
                    LocalHook.GetProcAddress("comdlg32.dll", "GetSaveFileNameW"),
                    new DGetSaveFileName(GetSaveFileName_Hooked),
                    this);

                SaveFileAsHook.ThreadACL.SetExclusiveACL(new Int32[] { 0 });
            }
            catch (Exception ExtInfo)
            {
                Interface.ReportException(ExtInfo);
                return;
            }

            Interface.IsInstalled(RemoteHooking.GetCurrentProcessId());

            RemoteHooking.WakeUpProcess();

            // wait for host process termination...
            try
            {
                while (true)
                {
                    Thread.Sleep(500);

                    
                    if (false)
                    {
                    }
                    else
                        Interface.Ping();
                    
                }
            }
            catch
            {
                // Ping() will raise an exception if host is unreachable
            }
        }

        public unsafe struct OPENFILENAME
        {
            public int structSize;
            public uint dlgOwner;
            public uint instance;

            public char* filter;
            public char* customFilter;
            public int maxCustFilter;
            public int filterIndex;

            public char* file;
            public int maxFile;

            public char* fileTitle;
            public int maxFileTitle;

            public char* initialDir;

            public char* title;

            public int flags;
            public short fileOffset;
            public short fileExtension;

            public char* defExt;

            public IntPtr custData;
            public IntPtr hook;

            public char* templateName;

            public IntPtr reservedPtr;
            public int reservedInt;
            public int flagsEx;
        }

        [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Unicode)]
        delegate bool DGetOpenFileName(ref OPENFILENAME ofn);
        [DllImport("comdlg32.dll", CharSet = CharSet.Unicode)]
        static extern bool GetOpenFileNameW(ref OPENFILENAME ofn);

        [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Unicode)]
        delegate bool DGetSaveFileName(ref OPENFILENAME ofn);
        [DllImport("comdlg32.dll", CharSet = CharSet.Unicode)]
        static extern bool GetSaveFileNameW(ref OPENFILENAME ofn);

        static bool GetOpenFileName_Hooked(ref OPENFILENAME ofn)
        {
            Main This = (Main)HookRuntimeInfo.Callback;

            String filter = "";
            String customFilter = "";
            String file = "";
            String initialDir = "";
            String title = "";
            String defExt = "";

            This.ParseOPENFILENAME(
                ref ofn,
                ref filter,
                ref customFilter,
                ref file,
                ref initialDir,
                ref title,
                ref defExt);

            String filename = This.Interface.OpenFileNameW(
                ofn.dlgOwner,
                filter,
                customFilter,
                file,
                initialDir,
                title,
                defExt);

            This.SetFilename(ref ofn, filename);

            if (filename.Equals(""))
                return GetOpenFileNameW(ref ofn);
            else
                return true;
        }

        static bool GetSaveFileName_Hooked(ref OPENFILENAME ofn)
        {
            Main This = (Main)HookRuntimeInfo.Callback;

            String filter = "";
            String customFilter = "";
            String file = "";
            String initialDir = "";
            String title = "";
            String defExt = "";

            This.ParseOPENFILENAME(
                ref ofn,
                ref filter,
                ref customFilter,
                ref file,
                ref initialDir,
                ref title,
                ref defExt);

            String filename = This.Interface.SaveFileNameW(
                ofn.dlgOwner,
                filter,
                customFilter,
                file,
                initialDir,
                title,
                defExt);

            This.SetFilename(ref ofn, filename);

            if (filename.Equals(""))
                return GetSaveFileNameW(ref ofn);
            else
                return true;
        }


        unsafe void SetFilename(ref OPENFILENAME ofn, String filename)
        {
            char[] fname = filename.ToCharArray();
            int i;
            for (i = 0; i < fname.Length; i++)
            {
                ofn.file[i] = fname[i];
            }
            ofn.file[i] = '\x00';
        }

        unsafe void ParseOPENFILENAME(
           ref OPENFILENAME ofn,
           ref String filter,
           ref String customFilter,
           ref String file,
           ref String initialDir,
           ref String title,
           ref String defExt)
        {
            if (ofn.filter != null)
            {
                int counter = 0;
                char last = '\x00';
                while (true)
                {
                    filter += ofn.filter[counter];

                    if (ofn.filter[counter] == '\x00' && last == '\x00')
                        break;

                    last = ofn.filter[counter];
                    counter++;
                }
            }

            if (ofn.customFilter != null)
            {
                int counter = 0;
                char last = '\x00';
                while (true)
                {
                    customFilter += ofn.customFilter[counter];

                    if (ofn.customFilter[counter] == '\x00' && last == '\x00')
                        break;

                    last = ofn.customFilter[counter];
                    counter++;
                }
            }

            if (ofn.file != null)
            {
                for (int i = 0; ofn.file[i] != '\x00'; i++)
                    file += ofn.file[i];
            }

            initialDir = "";
            if (ofn.initialDir != null)
            {
                for (int i = 0; ofn.initialDir[i] != '\x00'; i++)
                    initialDir += ofn.initialDir[i];
            }

            if (ofn.title != null)
            {
                for (int i = 0; ofn.title[i] != '\x00'; i++)
                    title += ofn.title[i];
            }

            if (ofn.defExt != null)
            {
                for (int i = 0; ofn.defExt[i] != '\x00'; i++)
                    defExt += ofn.defExt[i];
            }

        }
    }
}

Aug 20, 2009 at 9:52 AM

Does anyone know what there happens internally as it goes like this:

1. Execution comes to hooking function. Everything OK.

2. The calling process is non-responsive as execution stays on function.

3. If the execution is directed to original function with return statement original dialog works correctly.

I have also used other hooking libraries and haven't noticed anything like this happening.

 

Jun 30, 2011 at 8:50 PM

Did you ever resolve this ?

Jul 1, 2011 at 7:35 AM
Edited Jul 1, 2011 at 7:45 AM

Reasons why this happens comes obvious after studying some basic mechanisms of Windows operating system. It's based on messages that are send between UI components. For example in this case all user inputs create message for application and the application has to handle these messages or nothing happens. So it's like message pump that reads messages and passes them to actual handler. In this case this is done in the hooked thread so the application freezes. To get through this you have to create new thread and pass the messages to the handler yourself.

google for GetMessage and DispatchMessage functions

http://msdn.microsoft.com/en-us/library/ms644927%28v=vs.85%29.aspx