Created
February 28, 2018 00:44
-
-
Save seanrco/024e4461923eb8a3cc7d7b3be3428d37 to your computer and use it in GitHub Desktop.
CSV Importer that matches CSV column headers to a new object. This was developed by Adam and then retrofitted to SAIT's specific needs. I figure this is a good place to put this so we can collectively refine it.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
App_Data | |
|__xml | |
|__Custom | |
|__CSVImport | |
|__Controllers | |
|__ImportController.cs | |
|__Models | |
|__CSVResourceMap.cs | |
|__DocumentImporter.cs | |
|__Response.cs | |
|__scripts | |
|__app.html | |
|__loglister.html | |
|__logloader.html | |
|__Views | |
|__Import | |
|__Index.cshtml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<link rel="import" href="../bower_scripts/iron-flex-layout/iron-flex-layout.html"> | |
<link rel="import" href="../bower_scripts/iron-pages/iron-pages.html" /> | |
<link rel="import" href="../bower_scripts/paper-tabs/paper-tabs.html" /> | |
<link rel="import" href="../bower_scripts/app-layout/app-toolbar/app-toolbar.html"> | |
<link rel="import" href="../bower_scripts/app-layout/app-scroll-effects/app-scroll-effects.html"> | |
<link rel="import" href="../bower_scripts/app-layout/app-header/app-header.html"> | |
<link rel="import" href="./loglister.html"> | |
<link rel="import" href="./logloader.html"> | |
<dom-module id='csv-app'> | |
<style> | |
app-header { | |
position: fixed; | |
top: 0; | |
left: 0; | |
width: 100%; | |
height: 132px; | |
color: #fff; | |
background-color: #4285f4; | |
} | |
app-toolbar { | |
margin-bottom: 20px; | |
} | |
.page-content { | |
padding-top: 132px; | |
} | |
log-loader, log-lister { | |
overflow: hidden; | |
} | |
</style> | |
<template> | |
<app-header condenses reveals fixed effects="waterfall"> | |
<app-toolbar> | |
<h1 title>SAIT Course Catalogue Import</h1> | |
</app-toolbar> | |
<paper-tabs selected="{{tabDex}}"> | |
<paper-tab>Overview</paper-tab> | |
<paper-tab>Upload Resource</paper-tab> | |
<paper-tab>Logs</paper-tab> | |
</paper-tabs> | |
</app-header> | |
<iron-pages class="page-content" selected="{{tabDex}}"> | |
<div> | |
To upload an appropriately formatted CSV file, select the Upload Resources tab. Then drag and drop your CSV file into the drop area. The upload will begin immediately. | |
</div> | |
<div><log-loader uploadurl="[[uploadurl]]" url="[[statusurl]]"></log-loader></div> | |
<div><log-lister url="[[logurl]]"></log-lister></div> | |
</iron-pages> | |
</template> | |
<script> | |
Dropzone.autoDiscover = false; | |
Polymer({ | |
is: 'csv-app', | |
properties: { | |
statusurl: { | |
type: String, | |
value: "" | |
}, | |
logurl: { | |
type: String, | |
value: "" | |
}, | |
uploadurl: { | |
type: String, | |
value: "" | |
}, | |
tabDex: { | |
type: Number, | |
value: 0 | |
} | |
}, | |
ready: function () { | |
console.log(this); | |
} | |
}); | |
</script> | |
</dom-module> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using CsvHelper.Configuration; | |
using Ingeniux.CMS.Enums; | |
using System; | |
using System.Collections.Generic; | |
using System.Linq; | |
namespace Ingeniux.CMS.Applications | |
{ | |
public class CSVPageWrapper | |
{ | |
private const string PAGE_ASSIGN_REASON = "Assigning for CSV update"; | |
private const string OUTPUT_FOLDER = "Custom/"; | |
private const string OUTPUT_EXTENSION = "igx"; | |
private const string OUTPUT_DELIMITER = "~"; | |
CSVPageWrapper() | |
{ | |
} | |
public string Name { get; set; } | |
public string StartDate { get; set; } | |
public string EndDate { get; set; } | |
public char ConcatDelimiter | |
{ | |
get; | |
set; | |
} | |
public string Path | |
{ | |
get | |
{ | |
int stopIndex = Math.Max(0, Elements["CourseCode"].IndexOfAny("0123456789".ToCharArray())); | |
return Elements["CourseCode"].Substring(0, stopIndex); | |
} | |
} | |
public Dictionary<string, string> Elements { get; set; } | |
public List<string> Categories { get; set; } | |
public IPage CreatePage(IPage parentPage, Schema schema) | |
{ | |
try | |
{ | |
IPage page = parentPage.Site.CreatePage(schema, Name, parentPage); | |
var firstOrDefault = page.AllElements().FirstOrDefault(e => e.Name == "SiteControl"); | |
if (firstOrDefault != null) | |
{ | |
firstOrDefault.Value = "x4"; | |
} | |
this.UpdatePage(page); | |
return page; | |
} | |
catch (Exception e) | |
{ | |
throw new Exception("Page Creation Error"); | |
} | |
} | |
public void UpdatePage(IPage page) | |
{ | |
string xmlPath = page.Site.ContentStore.XmlDirectoryPath; | |
if (page.CheckedOut) | |
{ | |
page.CheckInSingleWithNoValidate(null); | |
} | |
page.CheckOut(false); | |
page.AssignUser(page.Site.CurrentUser, PAGE_ASSIGN_REASON); | |
//Setting a link on every element | |
var linkFirstOrDefault = page.AllElements().FirstOrDefault(e => e.Name == "RegisterLink") as LinkElement; | |
if (linkFirstOrDefault != null) | |
{ | |
linkFirstOrDefault.URL = | |
"http://register.sait.ca/servlet/CourseController?calendarType=All&method=getCourseContent&courseCode=" + | |
this.Elements["UrlCode"]; | |
linkFirstOrDefault.LinkType = EnumLinkElementType.IGX_EXTERNAL; | |
linkFirstOrDefault.SetAttributeValue("Target", "_blank"); | |
linkFirstOrDefault.SetAttributeValue("Link", "Click to see course offerings."); | |
} | |
foreach (var ele in this.Elements) | |
{ | |
var element = page.AllElements().FirstOrDefault(e => e.Name == ele.Key); | |
if (element != null) | |
{ | |
var _value = ele.Value; | |
if (element.Type == Enums.EnumElementType.IGX_MULTI_SELECT || element.Type == Enums.EnumElementType.IGX_ENUMERATION) | |
{ | |
string fileName = String.Format("{0}.{1}", element.Name, OUTPUT_EXTENSION); | |
string outputPath = System.IO.Path.Combine(xmlPath, OUTPUT_FOLDER, fileName); | |
string delimiter = string.Empty; | |
bool hasValue = false; | |
string output; | |
if (System.IO.File.Exists(outputPath)) | |
{ | |
IEnumerable<string> values = System.IO.File.ReadAllText(outputPath).Split('~'); | |
if (!values.Contains(ele.Value)) | |
{ | |
delimiter = values.Count() > 0 ? OUTPUT_DELIMITER : string.Empty; | |
} | |
else | |
{ | |
hasValue = true; | |
} | |
} | |
if (!hasValue) | |
{ | |
output = String.Format("{0}{1}", delimiter, ele.Value); | |
System.IO.File.AppendAllText(outputPath, output); | |
} | |
} | |
if (element.Type == Enums.EnumElementType.IGX_DOCUMENT) | |
{ | |
if (ele.Value.StartsWith("Documents/")) | |
{ | |
_value = ele.Value.SubstringAfter("Documents/"); | |
} | |
} | |
element.Value = _value; | |
} | |
} | |
foreach (var catName in this.Categories) | |
{ | |
int count; | |
var cat = page.Site.Session.TaxonomyManager.Categories(out count, nameInitialLetters: catName).Where(c => c.Name == catName); | |
if (cat != null && cat.Count() > 0) | |
{ | |
page.AddCategories(cat); | |
} | |
} | |
page.Name = Name; | |
try | |
{ | |
page.StartDate = DateTime.Parse(StartDate); | |
} | |
catch (Exception e) | |
{ | |
page.StartDate = null; | |
} | |
try | |
{ | |
page.EndDate = DateTime.Parse(EndDate); | |
} | |
catch (Exception e) | |
{ | |
page.EndDate = null; | |
} | |
} | |
} | |
public sealed class CSVResourceMap : CsvClassMap<CSVPageWrapper> | |
{ | |
public string Name = "Name"; | |
public string StartDate = "Available Date"; | |
public string EndDate = "Expiration Date"; | |
public string[][] colNames = | |
{ | |
new string[]{"Title","Name"}, | |
new string[]{"CourseCode","Prefix", "Code"}, | |
new string[]{"CourseDescription","Course Description:"}, | |
new string[]{"Credits","credits"}, | |
new string[]{"Prereqs","Prerequisite(s): (Rendered)"}, | |
new string[]{"Offered","Course Type"}, | |
new string[] {"BrowserBarTitle", "Name"}, | |
new string[] {"MetaDescription", "CourseDescription"}, | |
new string[] {"UrlCode", "Prefix", "Code"}, | |
}; | |
public string[][] catNames = | |
{ | |
//new string[]{"Regions"}, | |
//new string[]{"Catalog Item Number"} | |
}; | |
public Dictionary<string, string> ConcatDelimiter = new Dictionary<string, string>() | |
{ | |
{"UrlCode", "-" } | |
}; | |
public CSVResourceMap() | |
{ | |
Map(m => m.Name).Name(Name); | |
Map(m => m.Elements).ConvertUsing(row => | |
{ | |
Dictionary<string, string> eles = new Dictionary<string, string>(); | |
foreach (var cols in colNames) | |
{ | |
string fieldValue; | |
eles[cols[0]] = String.Empty; | |
var delimit = ConcatDelimiter.Item(cols[0]) ?? ""; | |
for (var i = 1; i < cols.Length; i++) | |
{ | |
if (!row.TryGetField(cols[i], out fieldValue)) | |
{ | |
continue; | |
} | |
eles[cols[0]] = eles[cols[0]] + delimit + fieldValue; | |
} | |
eles[cols[0]] = eles[cols[0]].TrimStart(delimit.ToCharArray()); | |
} | |
return eles; | |
}); | |
Map(m => m.Categories).ConvertUsing(row => | |
{ | |
List<string> eles = new List<string>(); | |
foreach (var cols in catNames) | |
{ | |
var delimit = ConcatDelimiter.Item(cols[0]) ?? ""; | |
string fieldValue; | |
string catName = String.Empty; | |
for (var i = 0; i < cols.Length; i++) | |
{ | |
if (!row.TryGetField(cols[i], out fieldValue)) | |
{ | |
continue; | |
} | |
catName = catName + delimit + fieldValue; | |
} | |
eles.Add(catName.TrimStart(delimit.ToCharArray())); | |
} | |
return eles; | |
}); | |
Map(m => m.StartDate).Name(StartDate); | |
Map(m => m.EndDate).Name(EndDate); | |
Map(m => m.ConcatDelimiter).Default(ConcatDelimiter); | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Collections.Generic; | |
using System.IO; | |
using System.Linq; | |
using System.Net; | |
using System.Text.RegularExpressions; | |
using System.Web; | |
namespace Ingeniux.CMS.Applications | |
{ | |
public class DocumentImporter | |
{ | |
public static string SCHEMA_NAME = "Resource"; //move this to a config | |
public static string VERSION_DIRECTORY_NAME = "_versions"; | |
private string _root; | |
public DocumentImporter(string xmlPath) | |
{ | |
_root = xmlPath; | |
} | |
private void _VersionDocument(DirectoryInfo directoryInfo, string fileName) | |
{ | |
string versionFolderPath = Path.Combine(directoryInfo.FullName,VERSION_DIRECTORY_NAME,fileName); | |
string fullFilePath = Path.Combine(directoryInfo.FullName,fileName); | |
DirectoryInfo versionDirInfo = Directory.CreateDirectory(versionFolderPath); | |
int versionNumber = 0; | |
if (versionDirInfo.GetFiles().Length > 0) | |
{ | |
var regex = new Regex(@"^(\d+)\$"); | |
var list = versionDirInfo.GetFiles("*" + fileName) | |
.Select(f => | |
{ | |
if (f.Name.Length <= 2) | |
{ | |
return -1; | |
} | |
if (regex.IsMatch(f.Name)) | |
{ | |
return Int32.Parse(regex.Match(f.Name).Groups[1].Value); | |
} | |
return -1; | |
}); | |
if(list.Count() > 0) | |
{ | |
versionNumber = list.Max() + 1; | |
} | |
} | |
versionNumber = Math.Max(versionNumber, 1); | |
string versionFileName = String.Format(@"{0}${1}", versionNumber,fileName); | |
string fullVersionPath = Path.Combine(versionFolderPath, versionFileName); | |
System.IO.File.Move(fullFilePath, fullVersionPath); | |
} | |
public FileInfo DocumentImport(string url, string directoryPath, string fileName, bool createVersion = true) | |
{ | |
string absDirectoryPath = Path.Combine(_root, directoryPath); | |
DirectoryInfo dirInfo = Directory.CreateDirectory(absDirectoryPath); | |
using (WebClient myWebClient = new WebClient()) | |
{ | |
// Download the Web resource and save it into the current filesystem folder. | |
string path = Path.Combine(dirInfo.FullName, fileName); | |
if (System.IO.File.Exists(path) && createVersion) | |
{ | |
_VersionDocument(dirInfo, fileName); | |
} | |
try | |
{ | |
myWebClient.DownloadFile(url, path); | |
}catch(Exception e) | |
{ | |
return null; | |
} | |
return new FileInfo(path); | |
} | |
} | |
public FileInfo DocumentImport(string uriPath, string directoryPath, bool createVersion = true) | |
{ | |
string absDirectoryPath = Path.Combine(_root, directoryPath); | |
DirectoryInfo dirInfo = Directory.CreateDirectory(absDirectoryPath); | |
FileInfo fileInfo = new FileInfo(uriPath); | |
if (fileInfo.Exists) | |
{ | |
string path = Path.Combine(absDirectoryPath, fileInfo.Name); | |
if (System.IO.File.Exists(path) && createVersion) | |
{ | |
_VersionDocument(dirInfo, fileInfo.Name); | |
} | |
fileInfo.MoveTo(path); | |
return fileInfo; | |
} | |
return null; | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using CsvHelper; | |
using Ingeniux.CMS.Applications; | |
using Ingeniux.CMS.RavenDB.Indexes; | |
using Ionic.Zip; | |
using MoreLinq; | |
using NLog; | |
using Raven.Client.Linq; | |
using System; | |
using System.Collections.Generic; | |
using System.ComponentModel.Composition; | |
using System.IO; | |
using System.Linq; | |
using System.Net; | |
using System.Threading; | |
using System.Threading.Tasks; | |
using System.Web; | |
using System.Web.Mvc; | |
namespace Ingeniux.CMS.Controller.Custom | |
{ | |
[Export(typeof(CMSControllerBase))] | |
[ExportMetadata("controller", "ImportController")] | |
[PartCreationPolicy(System.ComponentModel.Composition.CreationPolicy.NonShared)] | |
public class ImportController : CustomTabApplicationController | |
{ | |
public const string RESOURCE_FOLDER = "~/App_Data/xml/Custom/CSVImport/App_Data/ResourceImport"; | |
public const string LOGS_FILE = "~/App_Data/xml/Custom/CSVImport/App_Data/Logs.txt"; | |
public const string STAGING_FOLDER = "~/App_Data/xml/Custom/CSVImport/App_Data/ResourceImport/Staging"; | |
public const string COMP_SCHEMA_NAME = "CourseDetailPage"; | |
public const string COMP_ROOT_XID = "x49"; | |
public const string FOLDER_SCHEMA_NAME = "Folder"; | |
private static List<Log> _MessageBox = new List<Log>(); | |
private static Task _Task; | |
private static bool _Running = false; | |
private static ImporterTasks _CurrentTask = ImporterTasks.None; | |
private static string currentUser = ""; | |
public string StagingPath | |
{ | |
get | |
{ | |
return Server.MapPath(STAGING_FOLDER); | |
} | |
} | |
public string LogPath | |
{ | |
get | |
{ | |
return Server.MapPath(LOGS_FILE); | |
} | |
} | |
public string ResourcePath | |
{ | |
get | |
{ | |
return Server.MapPath(RESOURCE_FOLDER); | |
} | |
} | |
public ActionResult Index() | |
{ | |
bool isDebug = true; | |
#if DEBUG | |
isDebug = true; | |
#else | |
isDebug = false; | |
#endif | |
//System.Diagnostics.Debugger.Launch(); | |
var model = new CSVModel(); | |
model.IsDebug = isDebug; | |
return View(model); | |
} | |
static bool IsStalled { get; set; } | |
public JsonResult ImportStatus(int offset = 0) | |
{ | |
if (_Task == null) | |
{ | |
return Json(null); | |
} | |
if (_Task.IsCompleted && _CurrentTask != ImporterTasks.None) | |
{ | |
if (_Task.IsFaulted) | |
{ | |
_UpdateStatus(_Task.Exception.Message); | |
} | |
if (!IsStalled) | |
{ | |
IsStalled = true; | |
_UpdateStatus("Process Stalled."); | |
} | |
this._CleanUp(); | |
} | |
IEnumerable<Log> messages = _MessageBox.Skip(offset).Take(15); | |
var status = new StatusResponse(); | |
status.complete = _Task != null ? _Task.IsCompleted : false; | |
status.messages = messages; | |
status.currentTask = _CurrentTask; | |
status.totalMessageCount = _MessageBox.Count(); | |
return Json(status); | |
} | |
public JsonResult Logs(int offset = 0, int count = -1) | |
{ | |
IEnumerable<string> lines = null; | |
bool hasLock = false; | |
Monitor.Enter(_CleanLock, ref hasLock); | |
if (hasLock) | |
{ | |
if (!System.IO.File.Exists(LogPath)) | |
{ | |
System.IO.File.AppendAllLines(LogPath, new string[] { String.Empty }); | |
} | |
if (_Task != null && _Task.IsFaulted && _CurrentTask != ImporterTasks.None) | |
{ | |
_UpdateStatus(_Task.Exception.Message); | |
this._CleanUp(); | |
} | |
lines = System.IO.File.ReadLines(LogPath).Skip(offset).Where(s => !String.IsNullOrEmpty(s)).Take(25); | |
Monitor.Exit(_CleanLock); | |
} | |
return Json(lines); | |
} | |
private void _UpdateStatus(string messageText) | |
{ | |
bool hasLock = false; | |
Monitor.Enter(_CleanLock, ref hasLock); | |
if (hasLock) | |
{ | |
try | |
{ | |
var log = new Log(); | |
log.Message = messageText; | |
log.Time = DateTime.Now; | |
messageText = log.Time.ToFullDateTimeString() + messageText + ": " + _Common.CurrentUser.UserId; | |
System.IO.File.AppendAllLines(LogPath, new string[] { messageText }); | |
_Common.Log(LogLevel.Debug, messageText); | |
_MessageBox.Add(log); | |
} | |
catch (Exception e) | |
{ | |
throw new Exception("Log Broke"); | |
} | |
Monitor.Exit(_CleanLock); | |
} | |
} | |
private static Object _CleanLock = new Object(); | |
private void _CleanUp() | |
{ | |
bool hasLock = Monitor.TryEnter(_CleanLock); | |
if (hasLock) | |
{ | |
_UpdateStatus("Update Complete."); | |
Task.Run(() => | |
{ | |
Thread.Sleep(8000); | |
IsStalled = false; | |
_MessageBox = new List<Log>(); | |
_Running = false; | |
_CurrentTask = ImporterTasks.None; | |
Monitor.Exit(_CleanLock); | |
}); | |
} | |
} | |
private void _UpdateStagingFolder() | |
{ | |
_UpdateStatus("Updating Staging Folder"); | |
DirectoryInfo stagingDir = new DirectoryInfo(StagingPath); | |
DocumentImporter docImporter = new DocumentImporter(_Common.ContentStore.XmlDirectoryPath); | |
if (!stagingDir.Exists) | |
{ | |
try | |
{ | |
stagingDir = Directory.CreateDirectory(StagingPath); | |
} | |
catch (AggregateException ae) | |
{ | |
foreach (Exception e in ae.InnerExceptions) | |
{ | |
_UpdateStatus(e.Message); | |
} | |
} | |
catch (Exception e) | |
{ | |
_UpdateStatus(e.Message); | |
throw; | |
} | |
} | |
IEnumerable<FileInfo> stagedFiles = stagingDir.EnumerateFiles(); | |
IEnumerable<string> fileNames = stagedFiles.Select(f => f.Name); | |
_UpdateStatus(fileNames.Count() + " Files Staged"); | |
var nameBatch = fileNames.Batch(64); | |
List<Page> pages = new List<Page>(); | |
int count; | |
using (IUserWriteSession session = this._Common.ContentStore.OpenWriteSession(this._Common.CurrentUser)) | |
{ | |
foreach (var batch in nameBatch) | |
{ | |
IEnumerable<Page> pageBatch = (session.Site as Site).Query<Page, ISite, PagesByHierarchy>( | |
-1, | |
-1, | |
out count, | |
page => (page.SchemaName == COMP_SCHEMA_NAME && page._Name.In(batch) && !page.HierarchyValue.StartsWith(Site.RECYCLE_FOLDER_HIERARCHY)) | |
); | |
pages.AddRange(pageBatch); | |
} | |
_UpdateStatus(pages.Count() + " Pages Found"); | |
Dictionary<string, Page> pagesByName = pages.ToDictionaryDistinct(p => p.Name, p => p); | |
foreach (FileInfo staged in stagedFiles) | |
{ | |
try | |
{ | |
if (pagesByName.ContainsKey(staged.Name)) | |
{ | |
Page docPage = pagesByName[staged.Name]; | |
IElement pathElement = docPage.Element("Path"); | |
if (pathElement == null) | |
{ | |
continue; | |
} | |
string pathValue = pathElement.Value; | |
if (String.IsNullOrEmpty(pathValue)) | |
{ | |
continue; | |
} | |
_UpdateStatus(staged.FullName + " Importing"); | |
var file = docImporter.DocumentImport(staged.FullName, pathValue.Replace('/', '\\')); | |
if (file != null) | |
{ | |
_UpdateStatus("Versioned File: " + file.FullName); | |
} | |
_UpdateStatus(staged.FullName + " Imported"); | |
} | |
} | |
catch (Exception e) | |
{ | |
_UpdateStatus(e.Message); | |
} | |
} | |
} | |
} | |
private void _ImportZip(string path) | |
{ | |
_CurrentTask = ImporterTasks.Extracting; | |
try | |
{ | |
_UpdateStatus("Importing Zip: " + path); | |
if (!Directory.Exists(StagingPath)) | |
{ | |
Directory.CreateDirectory(StagingPath); | |
} | |
using (ZipFile zipFile = ZipFile.Read(path)) | |
{ | |
foreach (ZipEntry e in zipFile) | |
{ | |
e.Extract(StagingPath, ExtractExistingFileAction.OverwriteSilently); | |
} | |
//UpdateStagingFolder(); | |
} | |
_UpdateStatus("Imported Zip Complete: " + path); | |
} | |
catch (Exception e) | |
{ | |
_UpdateStatus(e.Message); | |
} | |
this._CleanUp(); | |
} | |
private void _ImportCSV(string path) | |
{ | |
_CurrentTask = ImporterTasks.ImportingDocuments; | |
try | |
{ | |
_UpdateStatus("Importing CSV: " + path); | |
TextReader textReader = new StreamReader(path); | |
List<CSVPageWrapper> pageWrappers = null; | |
using (var csv = new CsvReader(textReader)) | |
{ | |
csv.Configuration.WillThrowOnMissingField = false; | |
csv.Configuration.RegisterClassMap<CSVResourceMap>(); | |
pageWrappers = csv.GetRecords<CSVPageWrapper>().ToList(); | |
} | |
_UpdateStatus(pageWrappers.Count() + " Records found"); | |
if (pageWrappers == null) | |
{ | |
return; | |
} | |
Dictionary<string, IPage> folderDict = new Dictionary<string, IPage>(); | |
List<Page> pages = new List<Page>(); | |
IEnumerable<string> names = pageWrappers.Select(w => w.Name); | |
var nameBatch = names.Batch(64); | |
Dictionary<string, Page> existingPages = new Dictionary<string, Page>(); | |
using (IUserWriteSession session = this._Common.ContentStore.OpenWriteSession(this._Common.CurrentUser)) | |
{ | |
Schema schema = session.SchemasManager.SchemaByRootName(COMP_SCHEMA_NAME) as Schema; | |
Schema folderSchema = session.SchemasManager.SchemaByRootName(FOLDER_SCHEMA_NAME) as Schema; | |
if (schema == null) | |
{ | |
string message = "No Schema with the root name: " + COMP_SCHEMA_NAME; | |
_UpdateStatus(message); | |
throw new Exception(message); | |
} | |
int count; | |
foreach (var batch in nameBatch) | |
{ | |
IEnumerable<Page> pageBatch = (session.Site as Site).Query<Page, ISite, PagesByHierarchy>( | |
-1, | |
-1, | |
out count, | |
page => | |
page.SchemaName == COMP_SCHEMA_NAME && page._Name.In(batch) && | |
!page.HierarchyValue.StartsWith(Site.RECYCLE_FOLDER_HIERARCHY) | |
); | |
pages.AddRange(pageBatch); | |
} | |
existingPages = pages.ToDictionaryDistinct(p => p.Name, p => p); | |
} | |
var pageWrapperBatches = pageWrappers.Batch(32); | |
foreach (var pageWrapperBatch in pageWrapperBatches) | |
{ | |
foreach (var wrapper in pageWrapperBatch) | |
{ | |
using ( | |
IUserWriteSession session = | |
this._Common.ContentStore.OpenWriteSession(this._Common.CurrentUser)) | |
{ | |
IPage rootFolder = session.Site.Page(COMP_ROOT_XID); | |
Schema schema = session.SchemasManager.SchemaByRootName(COMP_SCHEMA_NAME) as Schema; | |
Schema folderSchema = session.SchemasManager.SchemaByRootName(FOLDER_SCHEMA_NAME) as Schema; | |
if (existingPages.ContainsKey(wrapper.Name)) | |
{ | |
_UpdateStatus("Updating Page: " + wrapper.Name); | |
var page = session.Site.Page(existingPages[wrapper.Name].Id); | |
wrapper.UpdatePage(page); | |
_UpdateStatus("Updated Page: " + wrapper.Name); | |
} | |
else | |
{ | |
rootFolder = session.Site.Page(COMP_ROOT_XID); | |
if (rootFolder == null) | |
{ | |
string message = "No page with xID: " + COMP_ROOT_XID; | |
_UpdateStatus(message); | |
throw new Exception(message); | |
} | |
IPage currentFolder = rootFolder; | |
IEnumerable<string> folderPath = new[] { wrapper.Path }; | |
string docPath = string.Empty; | |
foreach (string folderName in folderPath) | |
{ | |
if (!String.IsNullOrEmpty(docPath)) | |
{ | |
docPath += "/"; | |
} | |
docPath += folderName; | |
if (folderDict.ContainsKey(docPath)) | |
{ | |
currentFolder = folderDict[docPath]; | |
} | |
else | |
{ | |
//System.Diagnostics.Debugger.Launch(); | |
IPage childFolder; | |
int count; | |
childFolder = | |
currentFolder.Children(out count) | |
.Where(p => p.Name == folderName) | |
.FirstOrDefault(); | |
if (childFolder == null) | |
{ | |
_UpdateStatus("Creating Folder: " + docPath); | |
currentFolder = session.Site.CreatePage(folderSchema, folderName, | |
currentFolder); | |
_UpdateStatus("Created Folder: " + docPath + currentFolder.Id); | |
} | |
else | |
{ | |
currentFolder = childFolder; | |
} | |
folderDict[docPath] = currentFolder; | |
} | |
} | |
_UpdateStatus("Creating Component: " + docPath + wrapper.Name); | |
currentFolder = session.Site.Page(currentFolder.Id); | |
Page newPage = wrapper.CreatePage(currentFolder, schema) as Page; | |
_UpdateStatus("Created Component: " + docPath + newPage.Id); | |
if (newPage != null) | |
{ | |
pages.Add(newPage); | |
} | |
} | |
} | |
} | |
} | |
//_UpdateStagingFolder(); | |
_UpdateStatus("CSV Imported"); | |
} | |
catch (AggregateException ae) | |
{ | |
foreach (Exception e in ae.InnerExceptions) | |
{ | |
_UpdateStatus(e.Message); | |
} | |
} | |
catch (Exception e) | |
{ | |
_UpdateStatus(e.Message); | |
} | |
this._CleanUp(); | |
} | |
public JsonResult Upload(HttpPostedFileBase file) | |
{ | |
try | |
{ | |
if (file == null) | |
{ | |
this.Response.StatusCode = (int)HttpStatusCode.UnsupportedMediaType; | |
return Json(false); | |
} | |
ImportController._MessageBox = new List<Log>(); | |
_UpdateStatus("File Uploaded: " + file.FileName); | |
Response r = new Response(); | |
if ((_Task != null && !(_Task.IsCompleted || _Task.IsFaulted)) || _Running) | |
{ | |
_UpdateStatus("File Uploaded But task in progress: " + file.FileName); | |
r.success = false; | |
r.message = "task in process"; | |
return Json(r); | |
} | |
_Running = true; | |
if (file.ContentLength > 0) | |
{ | |
var fileName = Path.GetFileName(file.FileName); | |
if (!Directory.Exists(ResourcePath)) | |
{ | |
_UpdateStatus("Creating Folder: " + ResourcePath); | |
Directory.CreateDirectory(ResourcePath); | |
} | |
var path = Path.Combine(ResourcePath, fileName); | |
_UpdateStatus("Saving File: " + path); | |
file.SaveAs(path); | |
_UpdateStatus("Saved File: " + path); | |
if (fileName.EndsWith(".zip")) | |
{ | |
_Task = Task.Run(() => this._ImportZip(path)); | |
} | |
else | |
{ | |
_Task = Task.Run(() => this._ImportCSV(path)); | |
} | |
} | |
r.success = true; | |
return Json(r); | |
} | |
catch (AggregateException ae) | |
{ | |
foreach (Exception e in ae.InnerExceptions) | |
{ | |
_UpdateStatus(e.Message); | |
} | |
Response r = new Response(); | |
r.success = false; | |
r.message = "task in process"; | |
return Json(r); | |
} | |
catch (Exception e) | |
{ | |
_MessageBox = new List<Log>(); | |
_Running = false; | |
_CurrentTask = ImporterTasks.None; | |
_UpdateStatus(e.Message); | |
Response r = new Response(); | |
r.success = false; | |
r.message = "task in process"; | |
return Json(r); | |
} | |
} | |
public JsonResult GetUploads() | |
{ | |
return null; | |
} | |
public JsonResult LoadCSV(string FileName) | |
{ | |
return null; | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
@{ | |
Layout = null; | |
} | |
<!DOCTYPE html> | |
<html id="html"> | |
<head> | |
<title>Index</title> | |
<meta charset="utf-8"> | |
<meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1, user-scalable=no"> | |
<meta name="mobile-web-app-capable" content="yes"> | |
<meta name="apple-mobile-web-app-capable" content="yes"> | |
<script src="~/AppAsset/CSVImport/bower_scripts/webcomponentsjs/webcomponents-lite.min.js"></script> | |
@if (Model.IsDebug) | |
{ | |
<link rel="import" href="~/AppAsset/CSVImport/scripts/app.html" /> | |
} | |
else | |
{ | |
<link rel="import" href="~/AppAsset/CSVImport/scripts/app.min.html" /> | |
} | |
<style is="custom-style"> | |
html{ | |
overflow:hidden; | |
} | |
body { | |
margin: 0; | |
font-family: arial; | |
background-color: #eee; | |
} | |
</style> | |
</head> | |
<body unresolved> | |
<csv-app uploadurl="~/Apps/CSVImport/Import/Upload" logurl="~/Apps/CSVImport/Import/Logs" statusurl="~/Apps/CSVImport/Import/ImportStatus"></csv-app> | |
<script type="text/javascript"> | |
window.addEventListener("dragenter", function (e) { | |
e.preventDefault(); | |
e.dataTransfer.effectAllowed = "none"; | |
e.dataTransfer.dropEffect = "none"; | |
}, false); | |
window.addEventListener("dragover", function (e) { | |
e.preventDefault(); | |
e.dataTransfer.effectAllowed = "none"; | |
e.dataTransfer.dropEffect = "none"; | |
}); | |
window.addEventListener("drop", function (e) { | |
e.preventDefault(); | |
e.dataTransfer.effectAllowed = "none"; | |
e.dataTransfer.dropEffect = "none"; | |
}); | |
</script> | |
</body> | |
</html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<link rel="import" href="../bower_scripts/iron-ajax/iron-ajax.html" /> | |
<link rel="import" href="../bower_scripts/paper-card/paper-card.html"> | |
<link rel="import" href="../bower_scripts/iron-scroll-threshold/iron-scroll-threshold.html"> | |
<dom-module id='log-lister'> | |
<style> | |
paper-card { | |
margin-bottom: 16px; | |
width: 75%; | |
opacity: 0; | |
animation: fadein; | |
animation-fill-mode: forwards; | |
animation-duration: .3s; | |
text-align: left; | |
} | |
#logslist { | |
height: 535px; | |
overflow: auto; | |
margin-top: 16px; | |
} | |
.card-content { | |
word-wrap: break-word; | |
} | |
.cardRow { | |
text-align: center; | |
} | |
@keyframes fadein { | |
from { | |
opacity: 0; | |
} | |
to { | |
opacity: 1; | |
} | |
} | |
</style> | |
<template> | |
<iron-ajax id="ajaxlogs" | |
url="[[url]]" | |
body='{"offset":"[[offset]]"}' | |
handle-as="json" | |
method="POST" | |
debounce-duration="500" | |
last-response="{{lastResponse}}" | |
content-type="application/json" | |
on-response="processData"></iron-ajax> | |
<div id="logslist"> | |
<div> | |
<template is="dom-repeat" items="[[items]]"> | |
<div class="cardRow"> | |
<paper-card> | |
<div class="card-content">[[item]]</div> | |
</paper-card> | |
</div> | |
</template> | |
</div> | |
</div> | |
<iron-scroll-threshold id="scrollThreshold" lower-triggered="{{lowerTriggered}}" lower-threshold="20" on-lower-trigger="loadMoreData" fit> | |
</iron-scroll-threshold> | |
</template> | |
<script> | |
Polymer({ | |
is: 'log-lister', | |
properties: { | |
offset: { | |
type: Number, | |
value: 0 | |
}, | |
items: { | |
type: Array, | |
value: [] | |
}, | |
lastResponse: { | |
type: Object, | |
value:{} | |
}, | |
url: { | |
type: String, | |
value: "" | |
} | |
}, | |
timer:null, | |
ready: function () { | |
var scope = this; | |
this.$.scrollThreshold.scrollTarget = this.$.logslist; | |
this.timer = setInterval(function () { | |
if (scope.$.ajaxlogs.activeRequests.length == 0 && scope.items.length == 0) { | |
scope.loadMoreData(); | |
} | |
}, 2500); | |
}, | |
processData: function (evt) { | |
var scope = this; | |
if (!evt.detail.response) { | |
return; | |
} | |
var status = evt.detail.response; | |
for (var i in status) { | |
var message = status[i]; | |
//if (scope.items.indexOf(message) == -1) { | |
scope.offset++; | |
scope.push('items', message); | |
//} | |
} | |
if (this.timer && this.items.length > 0) { | |
clearInterval(this.timer); | |
} | |
}, | |
loadMoreData: function () { | |
var scope = this; | |
if (scope.$.ajaxlogs.activeRequests.length == 0) { | |
scope.$.ajaxlogs.generateRequest(); | |
} | |
scope.$.scrollThreshold.clearLower(); | |
} | |
}); | |
</script> | |
</dom-module> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<script type="text/javascript" src="../bower_scripts/dropzone/dist/min/dropzone.min.js"></script> | |
<link rel="stylesheet" href="../bower_scripts/dropzone/dist/min/dropzone.min.css" /> | |
<link rel="import" href="../bower_scripts/iron-ajax/iron-ajax.html" /> | |
<link rel="import" href="../bower_scripts/paper-card/paper-card.html"> | |
<dom-module id='log-loader'> | |
<style> | |
paper-card { | |
margin-bottom: 16px; | |
width: 75%; | |
opacity: 0; | |
animation: fadein; | |
animation-fill-mode: forwards; | |
animation-duration: .3s; | |
text-align: left; | |
} | |
#logs { | |
height: 360px; | |
overflow: auto; | |
margin-top: 16px; | |
} | |
.cardRow { | |
text-align: center; | |
} | |
.card-content { | |
word-wrap: break-word; | |
} | |
@keyframes fadein { | |
from { | |
opacity: 0; | |
} | |
to { | |
opacity: 1; | |
} | |
} | |
form{ | |
text-align:center; | |
} | |
</style> | |
<template> | |
<form action="[[uploadurl]]" method="post" class="dropzone" id="csvdropzone"></form> | |
<div id="logs"> | |
<div id="repeater"> | |
<template is="dom-repeat" items="[[items]]"> | |
<div class="cardRow"> | |
<paper-card heading="[[item.Time]]"> | |
<div class="card-content">[[item.Message]]</div> | |
</paper-card> | |
</div> | |
</template> | |
</div> | |
</div> | |
<iron-ajax id="ajax" | |
url="[[url]]" | |
body='{"offset":"[[offset]]"}' | |
handle-as="json" | |
content-type="application/json" | |
method="POST" | |
last-response="{{lastResponse}}" | |
on-response="processData"></iron-ajax> | |
<!-- scroll-target uses the document scroll --> | |
</template> | |
<script> | |
Polymer({ | |
is: 'log-loader', | |
properties: { | |
offset: { | |
type: Number, | |
value: 0 | |
}, | |
items: { | |
type: Array, | |
value: [] | |
}, | |
lastResponse: { | |
type: Object, | |
value:{} | |
}, | |
seen:{ | |
type: Object, | |
value:{} | |
}, | |
url:{ | |
type:String, | |
value:"" | |
}, | |
uploadurl: { | |
type:String, | |
value:"/default" | |
} | |
}, | |
timer:null, | |
ready: function () { | |
//this.$.scrollThreshold.scrollTarget = this.$.test; | |
var scope = this; | |
this.timer = setInterval(function () { | |
if (scope.$.ajax.activeRequests.length == 0) { | |
scope.$.ajax.generateRequest() | |
} | |
}, 1500); | |
if (window.igxDropzone) { | |
window.igxDropzone.url = this.uploadurl; | |
} | |
}, | |
cleared: false, | |
clear: function () { | |
var scope = this; | |
scope.seen = {}; | |
while (scope.items.length > 0) { | |
scope.pop('items'); | |
} | |
scope.offset = 0; | |
scope.cleared = false; | |
window.igxDropzone.enable(); | |
window.igxDropzone.removeAllFiles(); | |
setTimeout(function () { }, 0); | |
}, | |
unique : function(list){ | |
var u = {}, a = []; | |
for (var i = 0, l = list.length; i < l; ++i) { | |
if (u.hasOwnProperty(list[i])) { | |
continue; | |
} | |
a.push(list[i]); | |
u[list[i]] = 1; | |
} | |
return a; | |
}, | |
clearTimer:null, | |
processData: function (evt) { | |
var scope = this; | |
if (!evt.detail.response) { | |
return; | |
} | |
var status = evt.detail.response; | |
if (!scope.cleared && scope.items.length > 0 && status.currentTask == 0 && status.messages.length == 0) { | |
scope.cleared = true; | |
setTimeout(function () { scope.clear(); }, 7500); | |
} | |
for (var i in status.messages) { | |
var message = status.messages[i]; | |
if (status.currentTask != 0) { | |
window.igxDropzone.disable(); | |
} | |
var cleanTime = String(message.Time).replace(/^\/Date\(/g, ''); | |
cleanTime = String(cleanTime).replace(/\)\/$/g, ''); | |
message.TimeNumber = parseInt(cleanTime); | |
if (!scope.seen[message.TimeNumber]) { | |
if (this.clearTimer) { | |
clearInterval(this.clearTimer); | |
this.clear(); | |
} | |
scope.offset++; | |
message.Time = new Date(parseInt(cleanTime)).toLocaleDateString('en-us'); | |
scope.seen[message.TimeNumber] = true; | |
scope.unshift('items', message); | |
} | |
} | |
} | |
}); | |
window.addEventListener('WebComponentsReady', function csvload(e) { | |
window.removeEventListener("WebComponentsReady", csvload, false); | |
var form = document.querySelector("#csvdropzone"); | |
window.igxDropzone = new Dropzone(form, { | |
paramName: "file", | |
maxFilesize: 2000000, | |
clickable: true, | |
maxFiles: 1, | |
acceptedFiles: ".csv,.zip", | |
init: function () { | |
this.on("success", function () { this.disable() }); | |
} | |
}); | |
}); | |
</script> | |
</dom-module> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Collections.Generic; | |
using System.Linq; | |
using System.Web; | |
using System.Web.Mvc; | |
namespace Ingeniux.CMS.Applications | |
{ | |
public enum ImporterTasks | |
{ | |
None, | |
Extracting, | |
ImportingDocuments, | |
UpdatingCompoonents | |
} | |
public class Response | |
{ | |
public bool success; | |
public string message = String.Empty; | |
} | |
public class StatusResponse | |
{ | |
public bool complete = true; | |
public ImporterTasks currentTask = ImporterTasks.None; | |
public IEnumerable<Log> messages; | |
public int totalMessageCount; | |
} | |
public class Log | |
{ | |
public DateTime Time; | |
public string Message; | |
} | |
public class CSVModel | |
{ | |
public bool IsDebug = false; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment