/GetFilteredFileList.psd1 Secret
Last active
May 13, 2017 15:46
Star
You must be signed in to star a gist
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# | |
# Module manifest for module 'GetFilteredFileList' | |
# | |
# Generated by: Fredrik Kacsmarck | |
# | |
# Generated on: 2017-04-28 | |
# | |
@{ | |
# Script module or binary module file associated with this manifest. | |
RootModule = 'C:\Program Files\WindowsPowerShell\Modules\GetFilteredFileList\GetFilteredFileList.psm1' | |
# Version number of this module. | |
ModuleVersion = '1.0' | |
# Supported PSEditions | |
# CompatiblePSEditions = @() | |
# ID used to uniquely identify this module | |
GUID = '48ee758d-e3fc-4265-bb29-6dca4a4d6887' | |
# Author of this module | |
Author = 'Fredrik Kacsmarck' | |
# Company or vendor of this module | |
# CompanyName = '' | |
# Copyright statement for this module | |
Copyright = 'Fredrik Kacsmarck' | |
# Description of the functionality provided by this module | |
Description = 'Search for files with specific extension, a random extension of a certain length and or with file signature check' | |
# Minimum version of the Windows PowerShell engine required by this module | |
# PowerShellVersion = '' | |
# Name of the Windows PowerShell host required by this module | |
# PowerShellHostName = '' | |
# Minimum version of the Windows PowerShell host required by this module | |
# PowerShellHostVersion = '' | |
# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. | |
# DotNetFrameworkVersion = '' | |
# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. | |
# CLRVersion = '' | |
# Processor architecture (None, X86, Amd64) required by this module | |
# ProcessorArchitecture = '' | |
# Modules that must be imported into the global environment prior to importing this module | |
# RequiredModules = @() | |
# Assemblies that must be loaded prior to importing this module | |
# RequiredAssemblies = @() | |
# Script files (.ps1) that are run in the caller's environment prior to importing this module. | |
# ScriptsToProcess = @() | |
# Type files (.ps1xml) to be loaded when importing this module | |
# TypesToProcess = @() | |
# Format files (.ps1xml) to be loaded when importing this module | |
# FormatsToProcess = @() | |
# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess | |
# NestedModules = @() | |
# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. | |
FunctionsToExport = 'Get-FilteredFileList','Get-FileSignature','Get-FilesQuickIO' | |
# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. | |
CmdletsToExport = '*' | |
# Variables to export from this module | |
VariablesToExport = '*' | |
# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. | |
AliasesToExport = '*' | |
# DSC resources to export from this module | |
# DscResourcesToExport = @() | |
# List of all modules packaged with this module | |
# ModuleList = @() | |
# List of all files packaged with this module | |
# FileList = @() | |
# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. | |
PrivateData = @{ | |
PSData = @{ | |
# Tags applied to this module. These help with module discovery in online galleries. | |
# Tags = @() | |
# A URL to the license for this module. | |
# LicenseUri = '' | |
# A URL to the main website for this project. | |
# ProjectUri = '' | |
# A URL to an icon representing this module. | |
# IconUri = '' | |
# ReleaseNotes of this module | |
# ReleaseNotes = '' | |
} # End of PSData hashtable | |
} # End of PrivateData hashtable | |
# HelpInfo URI of this module | |
# HelpInfoURI = '' | |
# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. | |
# DefaultCommandPrefix = '' | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
Disclaimer: | |
The functions in this module are supplied AS IS, without any warranties or support. | |
I assume no responsibility or liability for the use of the functions. | |
Author: Fredrik Kacsmarck | |
E-mail: fredrik(at)chiloma.com | |
Blog: http://psfredrik.chiloma.com | |
Twitter: @psFredrik | |
Please leave a comment on the blog or on twitter if you find this usefull. | |
<# | |
<# | |
.Synopsis | |
Search file structure using QuickIO.Net library | |
.DESCRIPTION | |
Search the file structure without the 260+ character path length limit. | |
.EXAMPLE | |
Get-FilesQuickIO -FilePath <path> -Filter "*.*" -Recursive | |
.EXAMPLE | |
Get-FilesQuickIO -FilePath <path> -Filter "*.*" -Recursive -QuickIOPath ".\QuickIO\SchwabenCode.QuickIO.dll" | |
#> | |
function Get-FilesQuickIO | |
{ | |
[CmdletBinding()] | |
Param | |
( | |
# Path to start from | |
[Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true)] | |
[string]$FilePath, | |
# Path to start from | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[string]$QuickIOPath = "C:\Program Files\WindowsPowerShell\Modules\GetFilteredFileList\QuickIO\SchwabenCode.QuickIO.dll", | |
# To search recursively | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[switch]$Recursive, | |
# To filter | |
[Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true)] | |
[string]$Filter | |
) | |
try | |
{ | |
Write-Verbose "[Loading QuickIO.Net Library]: $QuickIOPath" | |
# Load QuickIO Assmebly | |
Add-Type -Path $QuickIOPath | |
} | |
catch | |
{ | |
Write-Error "There was an error loading the QuickIO.Net library, possibly the wrong path" | |
throw | |
} | |
# Initiate the file list | |
$fileList = @() | |
# Set search option based on recursive or not | |
if(($Recursive -eq $true) -and (($Filter -eq "*") -or ($Filter -eq "*.*"))) | |
{ | |
# Set the search option value | |
$searchOption = "AllDirectories" | |
# Get the list of files | |
$fileList = [SchwabenCode.QuickIO.QuickIODirectory]::EnumerateFiles([string]$FilePath,[string]$Filter,[System.IO.SearchOption]$searchOption) | |
} | |
elseif ($Recursive -eq $true) | |
{ | |
# Set the search option value | |
$searchOption = "AllDirectories" | |
# Set the directory pattern | |
$directoryPattern = "*" | |
# Get all the directories as the recursive option doesn't work when using a filter. | |
$directoryList = [SchwabenCode.QuickIO.QuickIODirectory]::EnumerateDirectories([string]$FilePath,[string]$directoryPattern,[System.IO.SearchOption]$searchOption) | select -ExpandProperty FullName | |
# Get any files from the selected path as only subdirectories will be included in the directorlist variable. | |
$fileList += try{[SchwabenCode.QuickIO.QuickIODirectory]::EnumerateFiles([string]$FilePath,[string]$Filter,[System.IO.SearchOption]$searchOption)}catch{$null} | |
# Loop through the directories with the file pattern | |
foreach($d in $directoryList) | |
{ | |
$fileList += try{[SchwabenCode.QuickIO.QuickIODirectory]::EnumerateFiles([string]$d,[string]$Filter,[System.IO.SearchOption]$searchOption)}catch{$null} | |
} | |
} | |
else{ | |
# Set the search option value | |
$searchOption = "TopDirectoryOnly" | |
# Get the list of files | |
$fileList = try{[SchwabenCode.QuickIO.QuickIODirectory]::EnumerateFiles([system.string]$FilePath,[system.string]$Filter,[System.IO.SearchOption]$searchOption)}catch{$null} | |
} | |
return $fileList | |
} | |
<# | |
.Synopsis | |
Search the supplied path for files matching extension and/or signature. | |
.DESCRIPTION | |
Search the path for files matching extension patterns. | |
You can base the search on a specific extension, a random extension by specifying the number of characters in the extension. | |
You can also select to verify if the file is using a specific file signature. | |
To find the file signature use the Get-FileSignature function. | |
Default settings can be found via Get-Help. | |
.EXAMPLE | |
Get-FilteredFileList -FilePath <path> -SpecificExtension "aaa" -Recursive | |
.EXAMPLE | |
Get-FilteredFileList -FilePath <path> -SpecificExtension "aaa","ccc" -Recursive | |
.EXAMPLE | |
Get-FilteredFileList -FilePath <path> -RandomExtension 6 -Recursive | |
.EXAMPLE | |
Get-FilteredFileList -FilePath <path> -RandomExtension 6 -ExcludeExtension "vsproj","csproj" -Recursive | |
.EXAMPLE | |
Get-FilteredFileList -FilePath <path> -FileSignature "25504446" -RandomExtension 6 -Recursive -QuickIOPath ".\QuickIO\SchwabenCode.QuickIO.dll" | |
#> | |
function Get-FilteredFileList | |
{ | |
[CmdletBinding()] | |
Param | |
( | |
# Path to start from | |
[Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true)] | |
[string]$FilePath, | |
# Path to the QuickIO Library | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[string]$QuickIOPath = "C:\Program Files\WindowsPowerShell\Modules\GetFilteredFileList\QuickIO\SchwabenCode.QuickIO.dll", | |
# Recursive search or not | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[switch]$Recursive, | |
# To match files against a file signature | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[string]$FileSignature, | |
# Search for specific extensions | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[string[]]$SpecificExtension, | |
# The amount of characters in the extension to match. | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[string]$RandomExtension, | |
# Any extensions to be excluded from the search. | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[string[]]$ExcludeExtension | |
) | |
Write-Verbose "[Verifying parameters]" | |
# Check if the File Signature and/or Specific Extension is specified or empty | |
$emptyFileSignature = [string]::IsNullOrEmpty($FileSignature) | |
$emptySpecificExtension = [string]::IsNullOrEmpty($SpecificExtension) | |
$emptyRandomExtension = [string]::IsNullOrEmpty($RandomExtension) | |
# Check if any of the search parameters are used. | |
if(($emptySpecificExtension) -and ($emptyRandomExtension) -and ($emptyFileSignature)) | |
{ | |
throw "None of the search parameters are used, use get-help for examples." | |
} | |
elseif(($emptySpecificExtension -eq $false) -and ($emptyRandomExtension -eq $false)) | |
{ | |
throw "You can't search for both a specific extension and a random extension at the same time" | |
} | |
# Progress bar to show to the user that the directories are gathered | |
Write-Progress -Activity "Creating list of files" -Status "Please wait." | |
# Get the files from the path and subdirectoriesm check if it's a recursive check or not | |
if($Recursive) | |
{ | |
$filesToCheck = Get-FilesQuickIO -FilePath $FilePath -Filter "*.*" -Recursive -QuickIOPath $QuickIOPath | |
} | |
else | |
{ | |
$filesToCheck = Get-FilesQuickIO -FilePath $FilePath -Filter "*.*" -QuickIOPath $QuickIOPath | |
} | |
# Check against search pattern | |
if(($emptySpecificExtension -eq $false)) | |
{ | |
$result = Search-FileExtension -FileList $filesToCheck -SpecificExtension $SpecificExtension -FileSignature $FileSignature -ExcludeExtension $ExcludeExtension | |
} | |
else | |
{ | |
$result = Search-FileExtension -FileList $filesToCheck -RandomExtension $RandomExtension -FileSignature $FileSignature -ExcludeExtension $ExcludeExtension | |
} | |
return $result | |
} | |
<# | |
.Synopsis | |
Helper function for searching the extension | |
.DESCRIPTION | |
Helper function for searching the extension | |
.EXAMPLE | |
Search-FileExtension -FileList <list of files> -SpecificExtension <specific extension> -FileSignature <filesignature> | |
.EXAMPLE | |
Search-FileExtension -FileList <list of files> -RandomExtension <number of characters> -FileSignature <filesignature> | |
#> | |
function Search-FileExtension | |
{ | |
[CmdletBinding()] | |
Param | |
( | |
# File List | |
[Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true)] | |
$FileList, | |
# To search for a specific extension | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
$SpecificExtension, | |
# To search for a number of characters in the extension. | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
$RandomExtension, | |
# FileSignature | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
$FileSignature, | |
# Excluded extension(s) | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
$ExcludeExtension | |
) | |
Write-Verbose "[Start processing file information]" | |
$result = @() | |
# Initialize counter and reference | |
$i = 1 | |
$fileListLength = $fileList.Length + 1 | |
foreach($f in $fileList) | |
{ | |
# Percent complete | |
[int]$percentComplete = $i / $fileListLength * 100 | |
if((($percentComplete % 10) -eq 0) -and ($percentComplete -ne $alreadyUpdated)) | |
{ | |
# Update progress | |
Write-Progress -Activity "Checking files..." -PercentComplete $percentComplete -CurrentOperation "$percentComplete% processed" -Status "Please wait." | |
$alreadyUpdated = $percentComplete | |
} | |
# Check wether a file signature have been supplied | |
if([string]::IsNullOrEmpty($FileSignature)) | |
{ | |
$fileSignatureSearch = $false | |
} | |
else | |
{ | |
$fileSignatureSearch = $true | |
} | |
# Extension from file list | |
$extension = $f.Name.Split(".") | select -last 1 | |
Write-Verbose "[Processing File]: $($f.name)" | |
# Call helper function to check if it's a match | |
$fileMatch = Test-FileMatch -SpecificExtension $SpecificExtension -RandomExtension $RandomExtension -ExcludeExtension $ExcludeExtension -Extension $extension | |
# Check the file signature if the option is set. | |
if(($fileMatch) -and ($fileSignatureSearch)) | |
{ | |
if(Test-FileSignature -FilePath $f.FullNameUnc -FileSignature $FileSignature) | |
{ | |
$result += $f | |
} | |
} | |
elseif($fileMatch) | |
{ | |
$result += $f | |
} | |
# Increase counter | |
$i++ | |
} | |
# Complete the progress bar | |
Write-Progress -Activity "Checking files..." -Completed -Status "All done." | |
Write-Verbose "[Processing file information completed]" | |
return $result | |
} | |
<# | |
.Synopsis | |
Helper function for checking the file signature | |
.DESCRIPTION | |
Helper function for checking the file signature | |
.EXAMPLE | |
Verify-FileSignature -FilePath c:\tmp\test.docx -FileSignature "504B0304" | |
#> | |
function Test-FileSignature | |
{ | |
[CmdletBinding()] | |
Param | |
( | |
# File List | |
[Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true)] | |
$FilePath, | |
# FileSignature | |
[Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true)] | |
[ValidatePattern('^\S*$')] | |
$FileSignature | |
) | |
# Get the amount of bytes in the signature | |
$signatureLength = $FileSignature.Length | |
# If the signature length is an odd number throw error | |
if(($signatureLength % 2) -eq 1) | |
{ | |
throw "The specified file signature have a length that is not even.`r`nPossible cause could be that the first byte is zero and need an extra zero" | |
} | |
Write-Verbose "[Get File Information]: $FilePath" | |
# Create QuickIO reference to the file, The QuickIO library should have been loading during the search. | |
$file = [SchwabenCode.QuickIO.QuickIOFileInfo]("$FilePath") | |
Write-Verbose "[Get File Signature]" | |
# Grab the signature byte length from file | |
[byte[]]$fileHeader = $file.GetFileChunks(($signatureLength/2)) | select -First 1 -ExpandProperty bytes | |
# Convert first bytes to HEX | |
foreach($f in $fileHeader) | |
{ | |
if(("{0:X}" -f $f).Length -eq 1) | |
{ | |
$hexHeader += "0{0:X}" -f $f | |
} | |
else | |
{ | |
$hexHeader += "{0:X}" -f $f | |
} | |
} | |
Write-Verbose "[Testing signature]: $hexHeader" | |
# Test the signature | |
if($hexHeader -eq $FileSignature) | |
{ | |
# Set the result to true | |
return $true | |
} | |
else | |
{ | |
# Set result to false | |
return $false | |
} | |
} | |
<# | |
.Synopsis | |
Gives the file signature in Hex. | |
.DESCRIPTION | |
Gives by default the file signature in Hex for the first 4 bytes. | |
Use the SignatureLength parameter to specify a different length. | |
For details about a signature or extension please look at the following links. | |
http://filext.com/ | |
http://filesignatures.net/ | |
.EXAMPLE | |
Get-FileSignature -FilePath c:\tmp\test.docx | |
.EXAMPLE | |
Get-FileSignature -FilePath c:\tmp\test.docx -SignatureLength 8 | |
#> | |
function Get-FileSignature | |
{ | |
[CmdletBinding()] | |
Param | |
( | |
# File List | |
[Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true)] | |
[string]$FilePath, | |
# File signature length | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
[int]$SignatureLength = 4, | |
# Path to QuickIO dll | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
$QuickIOPath = "C:\Program Files\WindowsPowerShell\Modules\GetFilteredFileList\QuickIO\SchwabenCode.QuickIO.dll" | |
) | |
try | |
{ | |
Write-Verbose "[Loading QuickIO Assembly]" | |
# Load QuickIO Assmebly | |
Add-Type -Path $QuickIOPath | |
} | |
catch | |
{ | |
Write-Error "There was an error loading the QuickIO.Net library, possibly the wrong path" | |
throw | |
} | |
Write-Verbose "[Get File Information]: $FilePath" | |
# Create QuickIO reference to the file | |
$file = [SchwabenCode.QuickIO.QuickIOFileInfo]("$FilePath") | |
Write-Verbose "[Get File Signature]" | |
# Grab the signature byte length from file | |
[byte[]]$fileHeader = $file.GetFileChunks(($signatureLength)) | select -First 1 -ExpandProperty bytes | |
# Convert first bytes to HEX | |
foreach($f in $fileHeader) | |
{ | |
if(("{0:X}" -f $f).Length -eq 1) | |
{ | |
$hexHeader += "0{0:X}" -f $f | |
} | |
else | |
{ | |
$hexHeader += "{0:X}" -f $f | |
} | |
} | |
Write-Verbose "[Signature returned]: $hexHeader" | |
return $hexHeader | |
} | |
<# | |
.Synopsis | |
Helper function to check if the file match the criterias | |
.DESCRIPTION | |
Helper function to check if the file match the criterias | |
.EXAMPLE | |
Test-FileMatch -SpecificExtension <"txt"> -RandomExtension <6> - | |
#> | |
function Test-FileMatch | |
{ | |
[CmdletBinding()] | |
Param | |
( | |
# To match for a specific extension | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
$SpecificExtension, | |
# To match on the number of characters in the extension | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
$RandomExtension, | |
# Excluded extension(s) | |
[Parameter(Mandatory=$false,ValueFromPipelineByPropertyName=$true)] | |
$ExcludeExtension, | |
# Extension | |
[Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true)] | |
$Extension | |
) | |
# Check wether it's a specificExtension, if not it's a random search | |
if([string]::IsNullOrEmpty($SpecificExtension)) | |
{ | |
$randomSearch = $true | |
} | |
else | |
{ | |
$randomSearch = $false | |
} | |
# If the extension is in the exclude list then return false | |
if($ExcludeExtension -notcontains $extension) | |
{ | |
# Go through file matches criterias | |
$matchSpecificExtension = $SpecificExtension -contains $extension | |
$matchRandomExtension = $extension -match "\b.{$RandomExtension}\b" | |
$matchAnyExtension = if((!$matchRandomExtension) -and (!$matchSpecificExtension) -and ($RandomExtension -eq "")){$true} else {$false} | |
if($matchAnyExtension) | |
{ | |
return $true | |
} | |
else | |
{ | |
# Just check if the file matches | |
if((($matchSpecificExtension) -and ($randomSearch -eq $false)) -or (($matchRandomExtension) -and ($randomSearch))) | |
{ | |
Write-Verbose "[Match File]: File matched criterias - $Extension" | |
return $true | |
} | |
else | |
{ | |
Write-Verbose "[Match File]: File didn't match criterias - $Extension" | |
return $false | |
} | |
} | |
} | |
else | |
{ | |
Write-Verbose "[Match File]: File were excluded - $Extension" | |
return $false | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment