Update: The new API required to make this work (SshConsole.GetInputStream(), SshConsole.GetOutputStream(), SshConsole.GetErrorStream() and FileServer.Settings.SshParameters.CompressionLevel) has been added to Rebex File Server 2017 R4: https://www.rebex.net/file-server/history.aspx#2017R4
You are right, Rebex File Server doesn't support executing external commands. Its simplified virtual shell is mostly supposed to complement SFTP and SCP protocols and all the built-in commands only work with the virtual file system.
However, what you are trying to do is a very interesting scenario and it actually turned out that adding support for it is quite simple - we just had to enhance the internals a bit and make the inner streams accessible through SshConsole, which is passed with ShellCommand event arguments.
You can give this a try as well - just sent you a link to the current beta build.
We have not sufficiently tested this yet, but I was able to clone a 300 MB repository without any issues by running hg clone ssh://demo@localhost:8022/repository_name against this proof-of-concept server app:
using System;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Rebex.Net;
using Rebex.Net.Servers;
namespace Rebex.Samples
{
    public class MercurialServer
    {
        // path to mercurial
        private const string MercurialPath = @"c:\Program Files\TortoiseHg\hg.exe";
        // path to a directory that contains the repositories
        private const string RepositoryRoot = @"c:\data\repository";
        // buffer size for data proxy
        private const int BufferSize = 32 * 1024;
        // log writer
        private static readonly ILogWriter LogWriter = new ConsoleLogWriter(LogLevel.Debug);
        // user name
        private const string UserName = "demo";
        // password
        private const string Password = "password";
        // server port
        private const int ServerPort = 8022;
        // server key
        private const string ServerKey = @"-----BEGIN RSA PRIVATE KEY-----
MIICXAIBAAKBgQDIkIOdSx47C4slUBVeEeVlCySg2IyT3laqBIr0uLlKIwwkI61F
Qen57kmgFG3Cs2zmbrnnRZYKR38EbrVyLH68wC9a1l6T7evUfnOD/O1fsPM1/XB6
m2shgUZsX6kXt1AUaPv2G4uH8J6Mk+RiJcrtBFslK0Lout5rLVm9Wn1d6QIDAQAB
AoGAYoBLC2S5l6EPOQeQPu+GHG5xEkfYHsUrBfwOLKtOYOE+lL8q2WFKYrOLWEHA
OEe7m55U0gckba74bDpdBZJhuT4MqaQ29PJn8fpi6RsFvvEqsgm6f9AWiScQ/n+h
/qSUraC5V8s03qF8XWuo5/87NNUunZDaA2UvUg8Urb0p87MCQQDjww8bwYVQS0lG
cpmjtNKqeLyfXtWPzo73/sCjIGsq/0Umdlufqlhezx3jvr+H/zaopp3A2dN6AKO8
nwFRrQrHAkEA4W48KB1R/Pjo5OefdiVAa6PFkmphFK0N7qjxg0WMCnxaqgbhwZFz
qnVq8RP2Al84Wn4fwpJwpKaqk82ZF2IhzwJBAMw1r+4q7OS5G9HWHnrxPZEq/7PE
y6ZMhVNFTmL0RiIfDlkV9cCKcwFOonX4KLI+2TsNaJPoufvBZw1PY1df1zECQCNt
CmEXcnn5t8e5KpMLeZswymye8RCpvWXDAOkrNb20Gx9bI4Ei1XV1LFAkXeWzhwyZ
g241SyRk2KuPhL5q+nsCQC8oiNdcTepTCGss+gNHlTdw2U5bhgUAZacCp5WgcEQJ
YNAx781J3Bwqh7s9vOCT0ubDxbi9+8uOui0ruL8AWA8=
-----END RSA PRIVATE KEY-----
";
        public static void Main()
        {
            var server = new FileServer();
            // enable logging
            server.LogWriter = LogWriter;
            // bind SSH shell subsystem to the specified port
            server.Bind(ServerPort, FileServerProtocol.Shell);
            // use the embedded private key for the sake of simplicity
            server.Keys.Add(new SshPrivateKey(new MemoryStream(Encoding.ASCII.GetBytes(ServerKey)), "password"));
            // only one user is supported for now
            server.Users.Add(UserName, Password);
            // custom command handler
            server.ShellCommand += (sender, e) =>
            {
                switch (e.Command)
                {
                    case "hg":
                        // intercept "hg" command
                        e.Action = RunMercurial;
                        break;
                    default:
                        // reject all other commands (optional)
                        e.WriteLine("Command not supported.");
                        e.ExitCode = 127;
                        break;
                }
            };
            server.Start();
            Console.WriteLine("Server is running, press any key to stop it.");
            Console.ReadKey(true);
        }
        public static int RunMercurial(string[] args, SshConsole console)
        {
            // start Mercurial
            var info = new ProcessStartInfo(MercurialPath, PrepareArguments(args));
            info.WorkingDirectory = RepositoryRoot;
            info.CreateNoWindow = true;
            info.UseShellExecute = false;
            info.RedirectStandardInput = true;
            info.RedirectStandardOutput = true;
            info.RedirectStandardError = true;
            var process = Process.Start(info);
            // proxy stdin/stdout/stderr
            Stream consoleInput = console.GetInputStream();
            Stream consoleOutput = console.GetOutputStream();
            Stream consoleError = console.GetErrorStream() ?? Stream.Null; // error stream is only available in terminal-less mode
            Stream processInput = process.StandardInput.BaseStream;
            Stream processOutput = process.StandardOutput.BaseStream;
            Stream processError = process.StandardError.BaseStream;
            CopyAllAsync("stdin", consoleInput, processInput);
            CopyAllAsync("stderr", processError, consoleError);
            CopyAllAsync("stdout", processOutput, consoleOutput);
            // wait for the process to end
            process.WaitForExit();
            // return the exit code
            return process.ExitCode;
        }
        private static string PrepareArguments(string[] args)
        {
            // TODO: This method is supposed to convert the argument array into a string,
            // but it is not yet implemented properly - if any of the arguments contains whitespaces or quotes,
            // this produces wrong results.
            // TODO: But if the users are only supposed to have SSH access to Mercurial repositories,
            // we should only allow "-R REPO_NAME serve --stdio" or its variants anyway.
            return string.Join(" ", args);
        }
        private static Task CopyAllAsync(string name, Stream input, Stream output)
        {
            // TODO: Error handling is missing.
            return Task.Run(() =>
            {
                byte[] buffer = new byte[BufferSize];
                while (true)
                {
                    int n = input.Read(buffer, 0, buffer.Length);
                    //Console.WriteLine("{0}: {1} bytes", name, n);
                    if (n == 0)
                    {
                        output.Close();
                        break;
                    }
                    output.Write(buffer, 0, n);
                    output.Flush();
                }
            });
        }
    }
}