gistfile1.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;

namespace CodeReview
{
    class Program
    {
        static void Main()
        {
            new TestFixtureA().Run();
            new TestFixtureB().Run();
        }

        abstract class TestFixture
        {
            protected readonly ConcurrentBag<CategoryBag> deserializeClasses =
                new ConcurrentBag<CategoryBag>();

            protected readonly ConcurrentQueue<Dictionary<string, double>> categoryClasses =
                new ConcurrentQueue<Dictionary<string, double>>();

            protected readonly ConcurrentQueue<ConcurrentDictionary<string, double>> result =
                new ConcurrentQueue<ConcurrentDictionary<string, double>>();

            public TestFixture()
            {
                foreach (var i in Enumerable.Range(1, 40))
                {
                    var bag = new HashSet<Word>();
                    var sb = new StringBuilder();
                    foreach (var j in Enumerable.Range(1, 20))
                    {
                        sb.Append('a');
                        bag.Add(new Word(sb.ToString(), 0, 0));
                    }

                    deserializeClasses.Add(new CategoryBag { Category = (new Category(i)).ToString(), Bag = bag });
                }

                foreach (var i in Enumerable.Range(1, 101))
                {
                    var dictionary = new Dictionary<string, double>();
                    var sb = new StringBuilder();
                    foreach (var j in Enumerable.Range(1, 1000))
                    {
                        sb.Append('a');
                        dictionary.Add(sb.ToString(), 0);
                    }

                    categoryClasses.Enqueue(dictionary);
                    result.Enqueue(new ConcurrentDictionary<string, double>());
                }
            }

            public abstract void Run();
        }

        class Category
        {
            private readonly int i;

            public Category(int i)
            {
                this.i = i;
            }

            public static explicit operator Category(int i)
            {
                return new Category(i);
            }
        }

        class Word
        {
            public Word(string word, int x, int y)
            {
            }
        }

        class CategoryBag
        {
            public string Category { get; set; }

            public HashSet<Word> Bag { get; set; }
        }

        class TestFixtureA : TestFixture
        {
            public override void Run()
            {
                var sw = Stopwatch.StartNew();
                foreach (var i in Enumerable.Range(1, 100))
                {
                    AddIDF(i);
                }
                sw.Stop();
                Console.WriteLine(sw.Elapsed);
            }

            void AddIDF(int i)
            {
                var category = ((Category)i).ToString();
                var bags = deserializeClasses.Where(clas => clas.Category == category)
                    .Select(clas => clas.Bag)
                    .ToList();
                var denominator = Convert.ToDouble(bags.Count);

                foreach (var word in categoryClasses.ElementAt(i))
                {
                    var key = word.Key;
                    var target = new Word(key, 0, 0);
                    var sum = bags.Count(bag => bag.Contains(target));
                    var temp = sum / denominator;

                    result.ElementAt(i)
                        .AddOrUpdate(key, temp, (s, d) => temp);
                }
            }
        }

        class TestFixtureB : TestFixture
        {
            public override void Run()
            {
                var sw = Stopwatch.StartNew();
                foreach (var i in Enumerable.Range(1, 100))
                {
                    AddIDF(new Nullable<int>(i));
                }
                sw.Stop();
                Console.WriteLine(sw.Elapsed);
            }

            void AddIDF(object index)
            {
                var i = index as int?;
                double sum;
                foreach (var word in categoryClasses.ElementAt(i.Value))
                {
                    sum =
                        deserializeClasses.Count(
                            clas =>
                            clas.Bag.Where(x => clas.Category == ((Category)i.Value).ToString())
                            .Contains(new Word(word.Key, 0, 0)));
                    var temp = Convert.ToDouble(sum) /
                        Convert.ToDouble(
                            deserializeClasses.Count(x => x.Category == ((Category)i.Value).ToString()));

                    result.ElementAt(i.Value).AddOrUpdate(word.Key, temp, (s, d) => temp);
                }
            }
        }
    }
}