Hangfire v1.4 Windows Server Configuration

README

This gist has the 3 main configuration files

HangfireService.cs and HangfireService.app.config are part of the windows service
HangfireJobStatusUpdateAttribute.cs is part of a common library that both the service and the web project use
Startup.cs is running in the MVC project

When I try to set up this configuration, scheduled tasks (in this case, a task every 15 minutes) does not execute and then multiple messages get queued while the service runs in the background.

HangfireJobStatusUpdateAttribute.cs

public class HangfireJobStatusUpdateAttribute : JobFilterAttribute, IElectStateFilter
{
    private IScheduledTasksRepository _repo;
    private IScheduledTaskLogRepository _log;
    private ITransactionFactory _tranFactory;
    private ICronService _cronSvc;

    /// <summary>
    /// Initializes a new instance of the <see cref="HangfireJobStatusUpdateAttribute"/> class.
    /// </summary>
    public HangfireJobStatusUpdateAttribute()
    {
        _repo = ObjectFactory.GetInstance<IScheduledTasksRepository>();
        _tranFactory = ObjectFactory.GetInstance<ITransactionFactory>();
        _cronSvc = ObjectFactory.GetInstance<ICronService>();
        _log = ObjectFactory.GetInstance<IScheduledTaskLogRepository>();
    }
    
    public void OnStateElection(ElectStateContext context)
    {
        ITransaction tran = null;
        try
        {
            var task = GetTask(context.JobId, ref tran);
            SetLogStatus(task, context, ref tran);
            if (task != null)
            {
                tran = _tranFactory.BuildITransaction("GetTask", IsolationLevel.ReadCommitted);
                task = GetTask(context.JobId, ref tran);
                if (context.CandidateState is EnqueuedState)
                {
                    var e = (EnqueuedState)context.CandidateState;
                    if (e.Reason != null && e.Reason.Contains("scheduler") && !(task.RunningJobId == null))
                    {
                        task.RunningJobId = context.JobId;
                        task.CurrentStatus = (byte)ScheduledTaskStatus.Queued;
                        _repo.Save(task, ref tran);                            
                    }
                }
                else if (context.CandidateState is ProcessingState)
                {
                    if (task.CurrentStatus == (byte)ScheduledTaskStatus.Running)
                    {
                        tran.Rollback();
                        BackgroundJob.Delete(context.JobId);
                    }
                    else
                    {
                        task.CurrentStatus = (byte)ScheduledTaskStatus.Running;
                        _repo.Save(task, ref tran);                            
                    }
                }
                else if (context.CandidateState is SucceededState)
                {
                    task.CurrentStatus = (byte)ScheduledTaskStatus.Ready;
                    task.LastRunDateTime = SystemTime.UtcNow();
                    task.RunningJobId = null;
                    task.NextRunDateTime = task.IsEnabled ? _cronSvc.GetNextScheduledOccurance(task.CronExpression, SystemTime.UtcNow()) : null;
                    task.LastStatus = "Successfully Executed";
                    _repo.Save(task, ref tran);                        
                }
                else if (context.CandidateState is FailedState)
                {
                    var failedState = (FailedState)context.CandidateState;
                    task.CurrentStatus = (byte)ScheduledTaskStatus.Ready;
                    task.LastRunDateTime = SystemTime.UtcNow();
                    task.RunningJobId = null;
                    task.LastStatus = string.Format("Job failed due to exception '{0}'", failedState.Exception);
                    task.NextRunDateTime = task.IsEnabled ? _cronSvc.GetNextScheduledOccurance(task.CronExpression, SystemTime.UtcNow()) : null;
                    _repo.Save(task, ref tran);                        
                }
                tran.Commit();
            }
        }
        finally
        {
            if (tran != null) tran.Dispose();
        }
    }

    private ScheduledTasksEntity GetTask(string jobId, ref ITransaction tran)
    {
        var name = JobStorage.Current.GetMonitoringApi().JobDetails(jobId).Job.Method.Name;
        if (name.Contains("Run15MinuteTasks"))
            return _repo.Get("15MinTasks", ApplicationSettingsController.GetEnvironment(), ref tran);
        else if (name.Contains("RunNightlyTasks"))
            return _repo.Get("NightlyTasks", ApplicationSettingsController.GetEnvironment(), ref tran);
        else if (name.Contains("RunOlap"))
            return _repo.Get("OLAP", ApplicationSettingsController.GetEnvironment(), ref tran);
        else
            return _repo.GetByJobID(jobId, ApplicationSettingsController.GetEnvironment(), ref tran);
    }

    private void SetLogStatus(ScheduledTasksEntity task, ElectStateContext context, ref ITransaction tran)
    {
        var log = new ScheduledTaskLogEntity()
        {
            HangfireJobID = context.JobId,
            DateTimeUTC = SystemTime.UtcNow(),
            Content = JsonConvert.SerializeObject(context)
        };
        if (task != null) log.ScheduledTaskID = task.ID;
        _log.Save(log, ref tran);
    }
}

HangfireService.app.config

<?xml version="1.0" encoding="utf-8"?>
<configuration>
    <configSections>
        <section name="sqlServerCatalogNameOverwrites" type="System.Configuration.NameValueFileSectionHandler, System, Version=1.0.5000.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"/>
        <section name="nlog" type="NLog.Config.ConfigSectionHandler, NLog"/>
    </configSections>
    <appSettings>
        <add key="strConn" value="Server=.\SQL2K8;Database=db;UID=sa;PWD=password"/>
        <add key="ServiceName" value="Hangfire Background Worker"/>
        <add key="ApplicationRoot" value="http://localhost/myapp" />
	      <add key="QueueBasePath" value=".\private$\samplecustomer_{0}"/>
        <add key="SynchronizationQueuePath" value=".\private$\synchronizationqueue"/>
        <add key="DocumentQueuePath" value=".\private$\documentqueue" />
        <add key="ReportGenerationQueuePath" value=".\private$\reportqueue" />
        <add key="EmailQueuePath" value=".\private$\emailqueue" />
        <add key="EnableOLAPProcessor" value="true"/>
        <add key="EnableGroupDPAProcessor" value="true"/>
        <add key="EnableMultipleWorkerProcessor" value="true"/>
        <add key="WorkersToEnable" value="5"/>
        <!-- Options: Production = 0, Staging = 1, Development = 2 -->
        <add key="Environment" value="2" />
    </appSettings>
    <startup>
        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/>
    </startup>
    <runtime>
        <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
            <dependentAssembly>
                <assemblyIdentity name="Common.Logging.Core" publicKeyToken="af08829b84f0328e" culture="neutral"/>
                <bindingRedirect oldVersion="0.0.0.0-3.0.0.0" newVersion="3.0.0.0"/>
            </dependentAssembly>
        </assemblyBinding>
    </runtime>
    <nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
        <targets>
            <target name="database" type="Database">
                <connectionString>
                    Server=.\SQL2K8;Database=db;UID=sa;PWD=password
                </connectionString>
                <commandText>
                    INSERT INTO Logging (Level, Logger, Message, MachineName, UserName, CallSite, Thread, Exception, Stacktrace) 
                    VALUES (@level, @logger, @message, @machinename, @user_name, @call_site, @threadid, @log_exception, @stacktrace);
                </commandText>
                <parameter name="@level" layout="${level}"/>
                <parameter name="@logger" layout="${logger}"/>
                <parameter name="@message" layout="${message}"/>
                <parameter name="@machinename" layout="${machinename}"/>
                <parameter name="@user_name" layout="${windows-identity:domain=true}"/>
                <parameter name="@call_site" layout="${callsite:filename=true}"/>
                <parameter name="@threadid" layout="${threadid}"/>
                <parameter name="@log_exception" layout="${exception}"/>
                <parameter name="@stacktrace" layout="${stacktrace}"/>
            </target>
        </targets>
        <rules>
            <logger name="*" minlevel="Trace" appendTo="database"/>
        </rules>
    </nlog>
</configuration>

HangfireService.cs

public partial class CognitionBackgroundWorker : ServiceBase
{
    private Thread _queueThread;
    private bool _stopping;
    private bool _initialized;

    private BackgroundJobServer _serverMultipleWorker;
    private BackgroundJobServer _serverAssignment;
    private BackgroundJobServer _serverGroupDPA;

    public CognitionBackgroundWorker()
    {
        InitializeComponent();
        ServiceName = ConfigurationManager.AppSettings["ServiceName"];
        eventLog1.Log = "Application";
        eventLog1.Source = ServiceName;
    }

    protected override void OnStart(string[] args)
    {
        eventLog1.WriteEntry(string.Format("{0} is starting.", ServiceName));
        _queueThread = new Thread(StartProcessing);
        _queueThread.Start();
    }

    private void StartProcessing()
    {
        _initialized = true;
        try
        {
            BootstrapService();
            AutoMapperConfig.Configure();
            JobStorage.Current = new SqlServerStorage(ConfigurationSettings.AppSettings["strConn"]);

            if (Boolean.Parse(ConfigurationManager.AppSettings["EnableOLAPProcessor"])) StartOLAPProcessor();
            if (Boolean.Parse(ConfigurationManager.AppSettings["EnableGroupDPAProcessor"])) StartGroupDPAProcessor();
            if (Boolean.Parse(ConfigurationManager.AppSettings["EnableMultipleWorkerProcessor"])) StartMultipleWorkerProcessor();
            GlobalJobFilters.Filters.Add(new AutomaticRetryAttribute { Attempts = 0 });
            SetScheduledTasks();
        }
        catch (Exception e)
        {
            _initialized = false;
            eventLog1.WriteEntry(string.Format("{0} was unable to start: '{1}'.", ServiceName, e.Message), EventLogEntryType.Error);
            throw;
        }
    }

    private void StartOLAPProcessor()
    {
        var storage = new SqlServerStorage(ConfigurationSettings.AppSettings["strConn"]);
        storage.UseMsmqQueues(ConfigurationSettings.AppSettings["QueueBasePath"], "assignment");
        var options = new BackgroundJobServerOptions()
        {
            Queues = new[] { "assignment" },
            WorkerCount = 1,
            ServerName = String.Format("{0}-{1}", Environment.MachineName, "assignment")
        };
        _serverAssignment = new BackgroundJobServer(options, storage);            
    }

    private void StartGroupDPAProcessor()
    {
        var storage = new SqlServerStorage(ConfigurationSettings.AppSettings["strConn"]);
        storage.UseMsmqQueues(ConfigurationSettings.AppSettings["QueueBasePath"], "groupdpa");
        var options = new BackgroundJobServerOptions()
        {
            Queues = new[] { "groupdpa" },
            WorkerCount = 1,
            ServerName = String.Format("{0}-{1}", Environment.MachineName, "groupdpa")
        };
        _serverGroupDPA = new BackgroundJobServer(options, storage);            
    }

    private void StartMultipleWorkerProcessor()
    {
        var storage = new SqlServerStorage(ConfigurationSettings.AppSettings["strConn"]);
        storage.UseMsmqQueues(ConfigurationSettings.AppSettings["QueueBasePath"], "multipleworker");
        var options = new BackgroundJobServerOptions()
        {
            Queues = new[] { "multipleworker" },
            WorkerCount = int.Parse(ConfigurationManager.AppSettings["WorkersToEnable"]),
            ServerName = String.Format("{0}-{1}", Environment.MachineName, "multipleworker")
        };
        _serverMultipleWorker = new BackgroundJobServer(options, storage);            
    }

    private void BootstrapService()
    {
        var container = new Container(x =>
        {
            x.Scan(scanner =>
            {
                scanner.TheCallingAssembly();
                scanner.WithDefaultConventions().OnAddedPluginTypes(t => t.HybridHttpOrThreadLocalScoped());
            });
        });

        JobActivator.Current = new StructureMapJobActivator(container);
    }

    private void SetScheduledTasks()
    {
        ITransaction tran = null;
        var tasks = ObjectFactory.GetInstance<IScheduledTaskFactory>().GetScheduledTasksByEnvironment(ApplicationSettingsController.GetEnvironment(), ObjectFactory.GetInstance<ITimeStampCreationService>().CreateSystemAccountTimeStamp(SystemTime.UtcNow()), ref tran).Tasks;
        var hfsvc = ObjectFactory.GetInstance<IHangfireService>();
        TimeZoneInfo tz = TimeZoneInfo.FindSystemTimeZoneById(ObjectFactory.GetInstance<ISystemVariableRepository>().GetValue((long)SystemVariable.System_Time_Zone));
        foreach (ScheduledTask task in tasks)
        {
            hfsvc.UpdateScheduledTaskInHangfire(task.HangfireCode, task.CronExpression, task.CommandLine, task.IsEnabled, tz);
        }
    }

    protected override void OnStop()
    {
        _stopping = true;
        if (_serverMultipleWorker != null) _serverMultipleWorker.Dispose();            
        if (_serverAssignment != null) _serverAssignment.Dispose();
        if (_serverGroupDPA != null) _serverGroupDPA.Dispose();                        
        _queueThread.Join(5000);
        eventLog1.WriteEntry(string.Format("{0} is stopping.", ServiceName));

    }

}

Startup.cs

public class Startup
{
    public void Configuration(IAppBuilder app)
    {            
        GlobalConfiguration.Configuration
            .UseSqlServerStorage(System.Configuration.ConfigurationSettings.AppSettings["strConn"])
            .UseMsmqQueues(ConfigurationSettings.AppSettings["QueueBasePath"], "assignment", "groupdpa", "multipleworker");                        
        GlobalJobFilters.Filters.Add(new AutomaticRetryAttribute { Attempts = 0 }); //do not attempt to retry the jobs automatically

        var options = new DashboardOptions { AppPath = VirtualPathUtility.ToAbsolute("~/admin/tasks") };
        app.UseHangfireDashboard("/admin/hangfire", options);
    }
}