IndexRebuilder.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using TACT.Net.Common;
using TACT.Net.Cryptography;

namespace TACT.Net.Indices
{
    internal class IndexRebuilder
    {
        private readonly IndexContainer _indexContainer;
        private readonly TACTRepo _repo;
        private readonly List<FileStream> _handles;

        public IndexRebuilder(IndexContainer indexContainer, TACTRepo repo)
        {
            _indexContainer = indexContainer;
            _repo = repo;
            _handles = new List<FileStream>();
        }

        public void Rebuild(string sourcedirectory, string directory)
        {
            Directory.CreateDirectory(directory);
            string tempBlobFilePath = Path.Combine(directory, "temp.blob");

            // clear the old archive references from the config
            _repo.ConfigContainer.CDNConfig.GetValues("archives").Clear();
            _repo.ConfigContainer.CDNConfig.GetValues("archives-index-size").Clear();

            var comparer = new MD5HashComparer();
            var entries = BuildEntries(sourcedirectory).OrderBy(x => comparer);

            var partitions = EnumerablePartitioner.ConcreteBatch(entries, IndexContainer.ArchiveDataSize, (x) => (uint)x.CompressedSize);
            foreach (var partition in partitions)
            {
                // create the blob
                WriteTempBlob(partition, tempBlobFilePath);

                IndexFile index = new IndexFile(IndexType.Data);
                index.Add(partition);
                index.Write(directory, _repo.ConfigContainer);

                // move the blob to it's real location
                string newpath = Helpers.GetCDNPath(index.Checksum.ToString(), "data", directory);
                File.Move(tempBlobFilePath, newpath);
            }

            ClearHandles();
        }

        /// <summary>
        /// Generates a new set of IndexEntries for the new archives
        /// </summary>
        /// <param name="sourcedirectory"></param>
        /// <returns></returns>
        private IEnumerable<IndexEntry> BuildEntries(string sourcedirectory)
        {
            ushort i = 0;
            foreach (var index in _indexContainer.DataIndices)
            {
                if (index.IsLooseIndex || index.IsGroupIndex)
                    continue;

                // open a handle to the existing blob
                string blob = Helpers.GetCDNPath(index.Checksum.ToString(), "data", sourcedirectory);
                _handles.Add(new FileStream(blob, FileMode.Open, FileAccess.Read, FileShare.Read));

                foreach (var entry in index.Entries)
                {
                    // skip unused entries
                    if (!_repo.EncodingFile.ContainsEKey(entry.Key))
                        continue;

                    // create a new entry and set it's handle ref
                    var tmp = entry.Clone();
                    tmp.IndexOrdinal = i;
                    yield return tmp;
                }

                i++;
            }

            yield break;
        }

        /// <summary>
        /// Creates the blob in a temporary location
        /// </summary>
        /// <param name="entries"></param>
        /// <param name="filename"></param>
        private void WriteTempBlob(IList<IndexEntry> entries, string filename)
        {
            using (var fs = File.Create(filename))
            {
                foreach (var entry in entries)
                {
                    // copy the data from the original archive
                    _handles[entry.IndexOrdinal].Position = entry.Offset;
                    _handles[entry.IndexOrdinal].PartialCopyTo(fs, (long)entry.CompressedSize);

                    // reset the handle ref
                    entry.IndexOrdinal = 0;
                }
            }
        }

        /// <summary>
        /// Closes all open handles to the blobs
        /// </summary>
        private void ClearHandles()
        {
            _handles.ForEach(x => x?.Dispose());
        }
    }
}