Redis Connection Multiplexing

.gitattributes

###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto

###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs     diff=csharp

###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following 
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
#*.sln       merge=binary
#*.csproj    merge=binary
#*.vbproj    merge=binary
#*.vcxproj   merge=binary
#*.vcproj    merge=binary
#*.dbproj    merge=binary
#*.fsproj    merge=binary
#*.lsproj    merge=binary
#*.wixproj   merge=binary
#*.modelproj merge=binary
#*.sqlproj   merge=binary
#*.wwaproj   merge=binary

###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
#*.jpg   binary
#*.png   binary
#*.gif   binary

###############################################################################
# diff behavior for common document formats
# 
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the 
# entries below.
###############################################################################
#*.doc   diff=astextplain
#*.DOC   diff=astextplain
#*.docx  diff=astextplain
#*.DOCX  diff=astextplain
#*.dot   diff=astextplain
#*.DOT   diff=astextplain
#*.pdf   diff=astextplain
#*.PDF   diff=astextplain
#*.rtf   diff=astextplain
#*.RTF   diff=astextplain

.gitignore

## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.

# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates

# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs

# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
build/
bld/
[Bb]in/
[Oo]bj/

# Visual Studio 2015 cache/options directory
.vs/

# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*

# NUNIT
*.VisualState.xml
TestResult.xml

# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c

# DNX
project.lock.json
artifacts/

*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc

# Chutzpah Test files
_Chutzpah*

# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opensdf
*.sdf
*.cachefile

# Visual Studio profiler
*.psess
*.vsp
*.vspx

# TFS 2012 Local Workspace
$tf/

# Guidance Automation Toolkit
*.gpState

# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user

# JustCode is a .NET coding add-in
.JustCode

# TeamCity is a build add-in
_TeamCity*

# DotCover is a Code Coverage Tool
*.dotCover

# NCrunch
_NCrunch_*
.*crunch*.local.xml

# MightyMoose
*.mm.*
AutoTest.Net/

# Web workbench (sass)
.sass-cache/

# Installshield output folder
[Ee]xpress/

# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html

# Click-Once directory
publish/

# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
## TODO: Comment the next line if you want to checkin your
## web deploy settings but do note that will include unencrypted
## passwords
#*.pubxml

*.publishproj

# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config

# Windows Azure Build Output
csx/
*.build.csdef

# Windows Store app package directory
AppPackages/

# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/

# Others
ClientBin/
[Ss]tyle[Cc]op.*
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs

# RIA/Silverlight projects
Generated_Code/

# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm

# SQL Server files
*.mdf
*.ldf

# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings

# Microsoft Fakes
FakesAssemblies/

# Node.js Tools for Visual Studio
.ntvs_analysis.dat

# Visual Studio 6 build log
*.plg

# Visual Studio 6 workspace options file
*.opt

# LightSwitch generated files
GeneratedArtifacts/
_Pvt_Extensions/
ModelManifest.xml

packages.config

<?xml version="1.0" encoding="utf-8"?>
<packages>
  <package id="StackExchange.Redis" version="1.0.488" targetFramework="net452" />
</packages>

Program.cs

using StackExchange.Redis;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;

namespace TestRedis
{
    class Program
    {
        static int s_counter = 0;
        static volatile string s_key = Guid.NewGuid().ToString();
        static List<ConnectionMultiplexer> muxes = new List<ConnectionMultiplexer>();
        static string s_prevKey = string.Empty;

        static void Main(string[] args)
        {
            ThreadPool.SetMinThreads(100, 1000);
            var masterConnectionMultiplexer = GetMultiplexer(Console.Out);
            UpdateMaster(masterConnectionMultiplexer);

            //Update the master periodically
            var timer = new Timer((state) =>
            {
                UpdateMaster(masterConnectionMultiplexer);
            }, null, TimeSpan.FromSeconds(0), TimeSpan.FromSeconds(5));


            // The master is updated it adds a new Guid key which should be visible to all clients. 
            // This will ensure that the clients always sees the data from the master instance. 
            List<Timer> timers = new List<Timer>(); // hand onto the timers. 
            for (int i = 0; i < 5; i++)
            {
                var childConnections = GetMultiplexer();
                var ensureKeyTimer = new Timer((state) =>
                {
                    EnsureKey(childConnections);
                }, null, TimeSpan.FromSeconds(0), TimeSpan.FromMilliseconds(1000));
                timers.Add(ensureKeyTimer);
            }

            Console.WriteLine("Press ENTER to terminate.");
            Console.ReadLine();
        }

        private static Random rng = new Random();
        private static int s_masterIndex = 0;

        static void UpdateMaster(ConnectionMultiplexer connection, TextWriter log = null)
        {

            try
            {
                var endpoints = connection.GetEndPoints();
                var next = rng.Next(endpoints.Length - 1);
                var server = connection.GetServer(endpoints[next]);
                server.MakeMaster(ReplicationChangeOptions.SetTiebreaker | ReplicationChangeOptions.Broadcast, log);
                Console.WriteLine("Making this server the master : " + server.EndPoint);

                // Adding a new key
                string newKey = Guid.NewGuid().ToString();
                connection.GetDatabase().StringSet(newKey, Interlocked.Increment(ref s_masterIndex).ToString() + "  " + DateTime.Now.ToString("hh:mm:ss:fff"));
                s_key = newKey;
            }
            catch (Exception exception)
            {
                Console.WriteLine("Exception " + exception.Message + "\n" + exception.StackTrace);
                Environment.Exit(0);
            }
        }


        public static ConnectionMultiplexer GetMultiplexer(TextWriter log = null)
        {
            ConfigurationOptions options = new ConfigurationOptions()
            {
                AbortOnConnectFail = true,
                AllowAdmin = true,
            };

            const int instanceCount = 10;
            const string remoteIp = "172.30.168.213";
            const int basePort = 6380;

            var addresses = Enumerable.Range(0, instanceCount)
                    .ToList()
                    .Shuffle()
                    .EnsureValidPorts()
                    .Select(i => remoteIp + ":" + (basePort + i));

            foreach (var address in addresses)
            {
                options.EndPoints.Add(address);
            }

            var multiplexer = ConnectionMultiplexer.Connect(options, log);
            muxes.Add(multiplexer);
            multiplexer.ConfigurationChangedBroadcast += (s, a) =>
            {
                Console.WriteLine(string.Format("Config changed Connection[{0}] - " + (a.EndPoint == null ? "(none)" : a.EndPoint.ToString()),
                                            muxes.IndexOf(multiplexer)));
            };

            return multiplexer;
        }

        public static void EnsureKey(ConnectionMultiplexer connection = null)
        {
            Interlocked.Increment(ref s_counter);
            bool shouldClose = false;
            if (connection == null)
            {
                connection = GetMultiplexer();
                shouldClose = true;
            }
            try
            {
                // We retry since the update master propogation could cause stale data
                // and the key might not be found. 
                int retryCount = 2;
                while (retryCount > 0)
                {
                    RedisValue returnValue = connection.GetDatabase().StringGet(s_key);
                    if (!String.IsNullOrEmpty(returnValue))
                    {
                        if (s_prevKey != s_key)
                        {
                            Console.WriteLine("****Key changed and new value = \"{0}\"", returnValue);
                            s_prevKey = s_key;
                        }

                        break;
                    }
                    else
                    {
                        retryCount--;
                        Thread.Sleep(100);
                    }
                }

                if (retryCount == 0)
                {
                    Console.WriteLine("Key not found in DB at iteration {0}", s_counter);
                    Environment.Exit(-1);
                }
            }
            finally
            {
                if (shouldClose)
                {
                    connection.Close(true);
                }
            }
        }
    }

    public static class ListExtensions
    {
        private static Random rng = new Random();

        public static List<T> Shuffle<T>(this List<T> list)
        {
            int n = list.Count;
            while (n > 1)
            {
                n--;
                int k = rng.Next(n + 1);
                T value = list[k];
                list[k] = list[n];
                list[n] = value;
            }

            return list;
        }

        public static List<int> EnsureValidPorts(this List<int> ints)
        {
            var ports = ints.OrderBy(e => e).ToArray();
            for (int i = 0; i < ints.Count; i++)
            {
                if (ports[i] != i)
                {
                    throw new InvalidOperationException("Incorrect shuffle.");
                }
            }

            return ints;
        }
    }
}

Redis-Standalone-Masters

Connecting to multiple master redis instances. 

TestRedis.csproj

<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
  <PropertyGroup>
    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
    <ProjectGuid>{43F746BE-7E49-4666-B6C6-E3A4D87FEB62}</ProjectGuid>
    <OutputType>Exe</OutputType>
    <AppDesignerFolder>Properties</AppDesignerFolder>
    <RootNamespace>TestRedis</RootNamespace>
    <AssemblyName>TestRedis</AssemblyName>
    <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
    <FileAlignment>512</FileAlignment>
    <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
    <PlatformTarget>AnyCPU</PlatformTarget>
    <DebugSymbols>true</DebugSymbols>
    <DebugType>full</DebugType>
    <Optimize>false</Optimize>
    <OutputPath>bin\Debug\</OutputPath>
    <DefineConstants>DEBUG;TRACE</DefineConstants>
    <ErrorReport>prompt</ErrorReport>
    <WarningLevel>4</WarningLevel>
    <Prefer32Bit>false</Prefer32Bit>
    <UseVSHostingProcess>false</UseVSHostingProcess>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
    <PlatformTarget>AnyCPU</PlatformTarget>
    <DebugType>pdbonly</DebugType>
    <Optimize>true</Optimize>
    <OutputPath>bin\Release\</OutputPath>
    <DefineConstants>TRACE</DefineConstants>
    <ErrorReport>prompt</ErrorReport>
    <WarningLevel>4</WarningLevel>
  </PropertyGroup>
  <ItemGroup>
    <Reference Include="StackExchange.Redis, Version=1.0.316.0, Culture=neutral, processorArchitecture=MSIL">
      <HintPath>..\packages\StackExchange.Redis.1.0.488\lib\net45\StackExchange.Redis.dll</HintPath>
      <Private>True</Private>
    </Reference>
    <Reference Include="System" />
    <Reference Include="System.Core" />
    <Reference Include="System.Xml.Linq" />
    <Reference Include="System.Data.DataSetExtensions" />
    <Reference Include="Microsoft.CSharp" />
    <Reference Include="System.Data" />
    <Reference Include="System.Net.Http" />
    <Reference Include="System.Xml" />
  </ItemGroup>
  <ItemGroup>
    <Compile Include="Program.cs" />
  </ItemGroup>
  <ItemGroup>
    <None Include="packages.config" />
  </ItemGroup>
  <ItemGroup>
    <Folder Include="Properties\" />
  </ItemGroup>
  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
       Other similar extension points exist, see Microsoft.Common.targets.
  <Target Name="BeforeBuild">
  </Target>
  <Target Name="AfterBuild">
  </Target>
  -->
</Project>

TestRedis.sln


Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 14
VisualStudioVersion = 14.0.23107.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestRedis", "TestRedis.csproj", "{43F746BE-7E49-4666-B6C6-E3A4D87FEB62}"
EndProject
Global
	GlobalSection(SolutionConfigurationPlatforms) = preSolution
		Debug|Any CPU = Debug|Any CPU
		Release|Any CPU = Release|Any CPU
	EndGlobalSection
	GlobalSection(ProjectConfigurationPlatforms) = postSolution
		{43F746BE-7E49-4666-B6C6-E3A4D87FEB62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
		{43F746BE-7E49-4666-B6C6-E3A4D87FEB62}.Debug|Any CPU.Build.0 = Debug|Any CPU
		{43F746BE-7E49-4666-B6C6-E3A4D87FEB62}.Release|Any CPU.ActiveCfg = Release|Any CPU
		{43F746BE-7E49-4666-B6C6-E3A4D87FEB62}.Release|Any CPU.Build.0 = Release|Any CPU
	EndGlobalSection
	GlobalSection(SolutionProperties) = preSolution
		HideSolutionNode = FALSE
	EndGlobalSection
EndGlobal