Last active
December 17, 2015 16:29
-
-
Save kijanawoodard/5638693 to your computer and use it in GitHub Desktop.
Slow Index Execute
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
RESULTS (in seconds): | |
starting 0.0004977 | |
products written 96.5585809 | |
indexed 430.3390117 | |
query complete - count matches 430.5240547 | |
index update applied 618.2628734 | |
done 618.2770718 | |
*/ | |
using System; | |
using System.Diagnostics; | |
using System.Linq; | |
using NUnit.Framework; | |
using Raven.Abstractions.Data; | |
using Raven.Abstractions.Indexing; | |
using Raven.Client.Indexes; | |
using Raven.Client.Linq.Indexing; | |
using Raven.Tests.Helpers; | |
namespace RavenTests | |
{ | |
public class IndexFailTests : RavenTestBase | |
{ | |
[Test] | |
public void CanUpdateAnIndexWithALotOfData() | |
{ | |
using (var store = NewDocumentStore()) | |
{ | |
var index1 = new ProductIndexOriginial().CreateIndexDefinition(); | |
store.DatabaseCommands.PutIndex("ProductIndex", index1); | |
var sw = Stopwatch.StartNew(); | |
Debug.WriteLine("starting " + sw.Elapsed.TotalSeconds); | |
var max = 500 * 1000; | |
using (var bulkInsert = store.BulkInsert(options: new BulkInsertOptions())) | |
{ | |
Enumerable.Range(1, max) | |
.Select(x => new Product() {Id = "products/" + x, Name = "Product " + x}) | |
.ToList() | |
.ForEach(bulkInsert.Store); | |
var stat = new ProductStat() | |
{ | |
Id = "products/1/stats", | |
ProductId = "products/1", | |
Likes = 1, | |
Buys = 0, | |
Created = DateTimeOffset.UtcNow | |
}; | |
bulkInsert.Store(stat); | |
} | |
Debug.WriteLine("products written " + sw.Elapsed.TotalSeconds); | |
WaitForIndexing(store); | |
Debug.WriteLine("indexed " + sw.Elapsed.TotalSeconds); | |
using (var session = store.OpenSession()) | |
{ | |
var count = session.Query<Product>("ProductIndex").Count(); | |
Assert.True(max == count); | |
} | |
Debug.WriteLine("query complete - count matches " + sw.Elapsed.TotalSeconds); | |
// var index2 = new ProductIndex().CreateIndexDefinition(); | |
// store.DatabaseCommands.PutIndex("ProductIndex", index2, true); | |
new ProductIndex().Execute(store); | |
Debug.WriteLine("index update applied " + sw.Elapsed.TotalSeconds); | |
using (var session = store.OpenSession()) | |
{ | |
var products = session.Query<Product>("ProductIndex").Take(10); | |
Assert.NotNull(products); | |
} | |
Debug.WriteLine("done " + sw.Elapsed.TotalSeconds); | |
} | |
} | |
public class Product | |
{ | |
public string Id { get; set; } | |
public string Name { get; set; } | |
public bool IsAvailableForPublicSearch { get; set; } //hook for search | |
public double TrendScore { get; set; } | |
} | |
public class ProductStat | |
{ | |
public string Id { get; set; } | |
public string ProductId { get; set; } | |
public int Likes { get; set; } | |
public int Buys { get; set; } | |
public DateTimeOffset Created { get; set; } | |
public bool IsAvailableForPublicSearch { get; set; } | |
} | |
public class ProductIndexOriginial : AbstractIndexCreationTask<Product> | |
{ | |
public ProductIndexOriginial() | |
{ | |
Map = products => from product in products | |
let stat = LoadDocument<ProductStat>(product.Id + "/stats") | |
let wants = stat == null ? 0 : Math.Max(stat.Likes * 10, 1) | |
let clicks = stat == null ? 0 : Math.Max(stat.Buys / 2, 1) | |
let score = stat == null ? 0 : Math.Log10(wants + clicks) | |
let epoch = new DateTimeOffset(2013, 1, 1, 0, 0, 0, TimeSpan.Zero) | |
let seconds = stat == null ? 0 : (stat.Created - epoch).TotalSeconds / 200 * 1000 | |
let trend = Math.Pow(Math.Round(score + seconds, 7), 0.25) | |
select new | |
{ | |
product.Name, | |
IsAvailableForPublicSearch = (stat == null || stat.IsAvailableForPublicSearch), | |
TrendScore = trend, | |
}.Boost((float)trend); | |
Index(x => x.Name, FieldIndexing.Analyzed); | |
Sort(x => x.TrendScore, SortOptions.Double); | |
} | |
} | |
public class ProductIndex : AbstractIndexCreationTask<Product> | |
{ | |
public ProductIndex() | |
{ | |
Map = products => from product in products | |
let stat = LoadDocument<ProductStat>(product.Id + "/stats") | |
let wants = stat == null ? 0 : Math.Max(stat.Likes * 10, 1) | |
let clicks = stat == null ? 0 : Math.Max(stat.Buys / 2, 1) | |
let score = stat == null ? 0 : Math.Log10(wants + clicks) | |
let epoch = new DateTimeOffset(2013, 1, 1, 0, 0, 0, TimeSpan.Zero) | |
let seconds = stat == null ? 0 : (stat.Created - epoch).TotalSeconds / 200 * 1000 | |
let trend = Math.Pow(Math.Round(score + seconds, 7), 0.25) | |
select new | |
{ | |
product.Name, | |
IsAvailableForPublicSearch = (stat == null || stat.IsAvailableForPublicSearch), | |
TrendScore = trend, | |
}.Boost((float)trend + 1); | |
Index(x => x.Name, FieldIndexing.Analyzed); | |
Sort(x => x.TrendScore, SortOptions.Double); | |
} | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Diagnostics; | |
using System.Linq; | |
using NUnit.Framework; | |
using Raven.Abstractions.Data; | |
using Raven.Abstractions.Indexing; | |
using Raven.Client.Indexes; | |
using Raven.Client.Linq.Indexing; | |
using Raven.Tests.Helpers; | |
namespace RavenTests | |
{ | |
public class IndexFailTests : RavenTestBase | |
{ | |
[Test] | |
public void CanUpdateAnIndexWithALotOfData() | |
{ | |
using (var store = NewDocumentStore()) | |
{ | |
var index1 = new ProductIndexOriginial().CreateIndexDefinition(); | |
store.DatabaseCommands.PutIndex("ProductIndex", index1); | |
var sw = Stopwatch.StartNew(); | |
Debug.WriteLine("starting " + sw.Elapsed.TotalSeconds); | |
var max = 500 * 1000; | |
using (var bulkInsert = store.BulkInsert(options: new BulkInsertOptions())) | |
{ | |
Enumerable.Range(1, max) | |
.Select(x => new Product() {Id = "products/" + x, Name = "Product " + x}) | |
.ToList() | |
.ForEach(bulkInsert.Store); | |
var stat = new ProductStat() | |
{ | |
Id = "products/1/stats", | |
ProductId = "products/1", | |
Likes = 1, | |
Buys = 0, | |
Created = DateTimeOffset.UtcNow | |
}; | |
bulkInsert.Store(stat); | |
} | |
Debug.WriteLine("products written " + sw.Elapsed.TotalSeconds); | |
WaitForIndexing(store); | |
Debug.WriteLine("indexed " + sw.Elapsed.TotalSeconds); | |
using (var session = store.OpenSession()) | |
{ | |
var count = session.Query<Product>("ProductIndex").Count(); | |
Assert.True(max == count); | |
} | |
Debug.WriteLine("query complete - count matches " + sw.Elapsed.TotalSeconds); | |
// var index2 = new ProductIndex().CreateIndexDefinition(); | |
// store.DatabaseCommands.PutIndex("ProductIndex", index2, true); | |
new ProductIndex().Execute(store); | |
Debug.WriteLine("index update applied " + sw.Elapsed.TotalSeconds); | |
using (var session = store.OpenSession()) | |
{ | |
var products = session.Query<Product>("ProductIndex").Take(10); | |
Assert.NotNull(products); | |
} | |
Debug.WriteLine("done " + sw.Elapsed.TotalSeconds); | |
} | |
} | |
public class Product | |
{ | |
public string Id { get; set; } | |
public string Name { get; set; } | |
public bool IsAvailableForPublicSearch { get; set; } //hook for search | |
public double TrendScore { get; set; } | |
} | |
public class ProductStat | |
{ | |
public string Id { get; set; } | |
public string ProductId { get; set; } | |
public int Likes { get; set; } | |
public int Buys { get; set; } | |
public DateTimeOffset Created { get; set; } | |
public bool IsAvailableForPublicSearch { get; set; } | |
public double Trend | |
{ | |
get | |
{ | |
var wants = Math.Max(Likes*10, 1); | |
var clicks = Math.Max(Buys/2, 1); | |
var score = Math.Log10(wants + clicks); | |
var epoch = new DateTimeOffset(2013, 1, 1, 0, 0, 0, TimeSpan.Zero); | |
var seconds = (Created - epoch).TotalSeconds/200*1000; | |
var trend = Math.Pow(Math.Round(score + seconds, 7), 0.25); | |
return trend; | |
} | |
} | |
} | |
public class ProductIndexOriginial : AbstractIndexCreationTask<Product> | |
{ | |
public ProductIndexOriginial() | |
{ | |
Map = products => from product in products | |
let stat = LoadDocument<ProductStat>(product.Id + "/stats") | |
select new | |
{ | |
product.Name, | |
IsAvailableForPublicSearch = (stat == null || stat.IsAvailableForPublicSearch), | |
TrendScore = stat.Trend, | |
}.Boost((float)stat.Trend); | |
Index(x => x.Name, FieldIndexing.Analyzed); | |
Sort(x => x.TrendScore, SortOptions.Double); | |
} | |
} | |
public class ProductIndex : AbstractIndexCreationTask<Product> | |
{ | |
public ProductIndex() | |
{ | |
Map = products => from product in products | |
let stat = LoadDocument<ProductStat>(product.Id + "/stats") | |
select new | |
{ | |
product.Name, | |
IsAvailableForPublicSearch = (stat == null || stat.IsAvailableForPublicSearch), | |
TrendScore = stat.Trend, | |
}.Boost((float)stat.Trend + 1); | |
Index(x => x.Name, FieldIndexing.Analyzed); | |
Sort(x => x.TrendScore, SortOptions.Double); | |
} | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
RESULTS (in seconds): | |
starting 0.0005486 | |
products written 99.4146726 | |
indexed 199.8829114 | |
query complete - count matches 200.0761762 | |
index update applied 200.2154348 | |
done 200.2238108 | |
*/ | |
using System; | |
using System.Diagnostics; | |
using System.Linq; | |
using NUnit.Framework; | |
using Raven.Abstractions.Data; | |
using Raven.Abstractions.Indexing; | |
using Raven.Client.Indexes; | |
using Raven.Client.Linq.Indexing; | |
using Raven.Tests.Helpers; | |
namespace RavenTests | |
{ | |
public class IndexFailTests : RavenTestBase | |
{ | |
[Test] | |
public void CanUpdateAnIndexWithALotOfData() | |
{ | |
using (var store = NewDocumentStore()) | |
{ | |
var index1 = new ProductIndexOriginial().CreateIndexDefinition(); | |
store.DatabaseCommands.PutIndex("ProductIndex", index1); | |
var sw = Stopwatch.StartNew(); | |
Debug.WriteLine("starting " + sw.Elapsed.TotalSeconds); | |
var max = 500 * 1000; | |
using (var bulkInsert = store.BulkInsert(options: new BulkInsertOptions())) | |
{ | |
Enumerable.Range(1, max) | |
.Select(x => new Product() {Id = "products/" + x, Name = "Product " + x}) | |
.ToList() | |
.ForEach(bulkInsert.Store); | |
var stat = new ProductStat() | |
{ | |
Id = "products/1/stats", | |
ProductId = "products/1", | |
Likes = 1, | |
Buys = 0, | |
Created = DateTimeOffset.UtcNow | |
}; | |
bulkInsert.Store(stat); | |
} | |
Debug.WriteLine("products written " + sw.Elapsed.TotalSeconds); | |
WaitForIndexing(store); | |
Debug.WriteLine("indexed " + sw.Elapsed.TotalSeconds); | |
using (var session = store.OpenSession()) | |
{ | |
var count = session.Query<Product>("ProductIndex").Count(); | |
Assert.True(max == count); | |
} | |
Debug.WriteLine("query complete - count matches " + sw.Elapsed.TotalSeconds); | |
// var index2 = new ProductIndex().CreateIndexDefinition(); | |
// store.DatabaseCommands.PutIndex("ProductIndex", index2, true); | |
new ProductIndex().Execute(store); | |
Debug.WriteLine("index update applied " + sw.Elapsed.TotalSeconds); | |
using (var session = store.OpenSession()) | |
{ | |
var products = session.Query<Product>("ProductIndex").Take(10); | |
Assert.NotNull(products); | |
} | |
Debug.WriteLine("done " + sw.Elapsed.TotalSeconds); | |
} | |
} | |
public class Product | |
{ | |
public string Id { get; set; } | |
public string Name { get; set; } | |
public bool IsAvailableForPublicSearch { get; set; } //hook for search | |
public double TrendScore { get; set; } | |
} | |
public class ProductStat | |
{ | |
public string Id { get; set; } | |
public string ProductId { get; set; } | |
public int Likes { get; set; } | |
public int Buys { get; set; } | |
public DateTimeOffset Created { get; set; } | |
public bool IsAvailableForPublicSearch { get; set; } | |
public double Trend | |
{ | |
get | |
{ | |
var wants = Math.Max(Likes*10, 1); | |
var clicks = Math.Max(Buys/2, 1); | |
var score = Math.Log10(wants + clicks); | |
var epoch = new DateTimeOffset(2013, 1, 1, 0, 0, 0, TimeSpan.Zero); | |
var seconds = (Created - epoch).TotalSeconds/200*1000; | |
var trend = Math.Pow(Math.Round(score + seconds, 7), 0.25); | |
return trend; | |
} | |
} | |
} | |
public class ProductIndexOriginial : AbstractIndexCreationTask<Product> | |
{ | |
public ProductIndexOriginial() | |
{ | |
Map = products => from product in products | |
select new | |
{ | |
product.Name, | |
}; | |
Index(x => x.Name, FieldIndexing.Analyzed); | |
Sort(x => x.TrendScore, SortOptions.Double); | |
} | |
} | |
public class ProductIndex : AbstractIndexCreationTask<Product> | |
{ | |
public ProductIndex() | |
{ | |
Map = products => from product in products | |
select new | |
{ | |
product.Name, | |
product.IsAvailableForPublicSearch, | |
}; | |
Index(x => x.Name, FieldIndexing.Analyzed); | |
Sort(x => x.TrendScore, SortOptions.Double); | |
} | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
RESULTS (in seconds): | |
starting 0.0005027 | |
products written 100.5340452 | |
put index 100.9752897 | |
indexed 484.9278777 | |
query complete - count matches 485.1127672 | |
index update applied 485.1294213 | |
done 485.1375714 | |
*/ | |
using System; | |
using System.Diagnostics; | |
using System.Linq; | |
using NUnit.Framework; | |
using Raven.Abstractions.Data; | |
using Raven.Abstractions.Indexing; | |
using Raven.Client.Indexes; | |
using Raven.Client.Linq.Indexing; | |
using Raven.Tests.Helpers; | |
namespace RavenTests | |
{ | |
public class IndexFailTests : RavenTestBase | |
{ | |
[Test] | |
public void CanUpdateAnIndexWithALotOfData() | |
{ | |
var sw = Stopwatch.StartNew(); | |
Debug.WriteLine("starting " + sw.Elapsed.TotalSeconds); | |
using (var store = NewDocumentStore()) | |
{ | |
// var index1 = new ProductIndexOriginial().CreateIndexDefinition(); | |
// store.DatabaseCommands.PutIndex("ProductIndex", index1); | |
// Debug.WriteLine("put index " + sw.Elapsed.TotalSeconds); | |
var max = 500 * 1000; | |
using (var bulkInsert = store.BulkInsert(options: new BulkInsertOptions())) | |
{ | |
Enumerable.Range(1, max) | |
.Select(x => new Product() {Id = "products/" + x, Name = "Product " + x}) | |
.ToList() | |
.ForEach(bulkInsert.Store); | |
var stat = new ProductStat() | |
{ | |
Id = "products/1/stats", | |
ProductId = "products/1", | |
Likes = 1, | |
Buys = 0, | |
Created = DateTimeOffset.UtcNow | |
}; | |
bulkInsert.Store(stat); | |
} | |
Debug.WriteLine("products written " + sw.Elapsed.TotalSeconds); | |
new ProductIndex().Execute(store); | |
Debug.WriteLine("put index " + sw.Elapsed.TotalSeconds); | |
WaitForIndexing(store); | |
Debug.WriteLine("indexed " + sw.Elapsed.TotalSeconds); | |
using (var session = store.OpenSession()) | |
{ | |
var count = session.Query<Product>("ProductIndex").Count(); | |
Assert.True(max == count); | |
} | |
Debug.WriteLine("query complete - count matches " + sw.Elapsed.TotalSeconds); | |
// var index2 = new ProductIndex().CreateIndexDefinition(); | |
// store.DatabaseCommands.PutIndex("ProductIndex", index2, true); | |
new ProductIndex().Execute(store); | |
Debug.WriteLine("index update applied " + sw.Elapsed.TotalSeconds); | |
using (var session = store.OpenSession()) | |
{ | |
var products = session.Query<Product>("ProductIndex").Take(10); | |
Assert.NotNull(products); | |
} | |
Debug.WriteLine("done " + sw.Elapsed.TotalSeconds); | |
} | |
} | |
public class Product | |
{ | |
public string Id { get; set; } | |
public string Name { get; set; } | |
public bool IsAvailableForPublicSearch { get; set; } //hook for search | |
public double TrendScore { get; set; } | |
} | |
public class ProductStat | |
{ | |
public string Id { get; set; } | |
public string ProductId { get; set; } | |
public int Likes { get; set; } | |
public int Buys { get; set; } | |
public DateTimeOffset Created { get; set; } | |
public bool IsAvailableForPublicSearch { get; set; } | |
public double Trend | |
{ | |
get | |
{ | |
var wants = Math.Max(Likes*10, 1); | |
var clicks = Math.Max(Buys/2, 1); | |
var score = Math.Log10(wants + clicks); | |
var epoch = new DateTimeOffset(2013, 1, 1, 0, 0, 0, TimeSpan.Zero); | |
var seconds = (Created - epoch).TotalSeconds/200*1000; | |
var trend = Math.Pow(Math.Round(score + seconds, 7), 0.25); | |
return trend; | |
} | |
} | |
} | |
public class ProductIndexOriginial : AbstractIndexCreationTask<Product> | |
{ | |
public ProductIndexOriginial() | |
{ | |
Map = products => from product in products | |
let stat = LoadDocument<ProductStat>(product.Id + "/stats") | |
let wants = stat == null ? 0 : Math.Max(stat.Likes * 10, 1) | |
let clicks = stat == null ? 0 : Math.Max(stat.Buys / 2, 1) | |
let score = stat == null ? 0 : Math.Log10(wants + clicks) | |
let epoch = new DateTimeOffset(2013, 1, 1, 0, 0, 0, TimeSpan.Zero) | |
let seconds = stat == null ? 0 : (stat.Created - epoch).TotalSeconds / 200 * 1000 | |
let trend = Math.Pow(Math.Round(score + seconds, 7), 0.25) | |
select new | |
{ | |
product.Name, | |
IsAvailableForPublicSearch = (stat == null || stat.IsAvailableForPublicSearch), | |
TrendScore = trend, | |
}.Boost((float)trend); | |
Index(x => x.Name, FieldIndexing.Analyzed); | |
Sort(x => x.TrendScore, SortOptions.Double); | |
} | |
} | |
public class ProductIndex : AbstractIndexCreationTask<Product> | |
{ | |
public ProductIndex() | |
{ | |
Map = products => from product in products | |
let stat = LoadDocument<ProductStat>(product.Id + "/stats") | |
let wants = stat == null ? 0 : Math.Max(stat.Likes * 10, 1) | |
let clicks = stat == null ? 0 : Math.Max(stat.Buys / 2, 1) | |
let score = stat == null ? 0 : Math.Log10(wants + clicks) | |
let epoch = new DateTimeOffset(2013, 1, 1, 0, 0, 0, TimeSpan.Zero) | |
let seconds = stat == null ? 0 : (stat.Created - epoch).TotalSeconds / 200 * 1000 | |
let trend = Math.Pow(Math.Round(score + seconds, 7), 0.25) | |
select new | |
{ | |
product.Name, | |
IsAvailableForPublicSearch = (stat == null || stat.IsAvailableForPublicSearch), | |
TrendScore = trend, | |
}.Boost((float)trend + 1); | |
Index(x => x.Name, FieldIndexing.Analyzed); | |
Sort(x => x.TrendScore, SortOptions.Double); | |
} | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment