I need to compare folders from multiple servers which do not have direct connection so i was trying to generate a text file with command dir /b /s >c:\server1.txt and feed them to script to compare.
my desired output is csv file in the following format
File name,Server1,server2,server3,is it missng file?
File1,yes,yes,yes,No
file2,Yes,Yes,No,Yes
File3,Yes,No,Yes,Yes
File4,No,Yes,Yes,Yes
these folders contains few hundred thousand files and the script is taking 4 to 5 hours to run the comparison. i thought threads will help to run fast using parallel processng but did not help. i have no expreience with threads and after searching for examples and implementing it was even slower than normal.
i probably am doing something wrong. Any help is much appreciated.
my desired output is csv file in the following format
File name,Server1,server2,server3,is it missng file?
File1,yes,yes,yes,No
file2,Yes,Yes,No,Yes
File3,Yes,No,Yes,Yes
File4,No,Yes,Yes,Yes
these folders contains few hundred thousand files and the script is taking 4 to 5 hours to run the comparison. i thought threads will help to run fast using parallel processng but did not help. i have no expreience with threads and after searching for examples and implementing it was even slower than normal.
i probably am doing something wrong. Any help is much appreciated.
The following is the script so far and it works fine but its taking long time.
[array]$contentArray = @()
[array]$allfileslist = @()
[array]$uniquefileslist = @()
[array]$allfilesSRVlist = @()
[array]$srvlist = @()
$FolderName = Split-Path -parent $MyInvocation.MyCommand.Definition
$reportName = $FolderName + "\ComparisonReport3.csv"
$ListOfFiles = get-childitem $FolderName
$List = $ListOfFiles | where {$_.extension -ieq ".txt"}
$list.count
$index = 0
foreach($listitem in $List){
$listfilename = $listitem.FullName
$listname = $listitem.Name
$listname = $listname.replace(".txt","")
$srvlist = $srvlist + $listname
write-host $listfilename
#$StrContent = get-content $listfilename
$StrContent = [io.file]::ReadAllLines($listfilename)
$contentArray += ,@($StrContent)
$StrContent.count
$contentArray[$index].count
$index = $index + 1
}
for($i = 0;$i -lt $index;$i++){
$allfileslist = $allfileslist + $contentArray[$i]
}
$allfileslist.count
$uniquefileslist = $allfileslist | sort-object | get-unique
$Stroutline = "File Name,"
foreach($srvlistitem in $srvlist){
$Stroutline = $Stroutline + $srvlistitem.ToUpper() + ","
}
$Stroutline = $Stroutline + "Is it Missing file?"
Write-Output $Stroutline | Out-File "$reportName" -Force
foreach($uniquefileslistitem in $uniquefileslist){
$Stroutline = ""
$missingfile = "No"
$Stroutline = $uniquefileslistitem + ","
for($i=0;$i -lt $index;$i++){
if($contentArray[$i] -contains $uniquefileslistitem){
$Stroutline = $Stroutline + "Yes,"
}
else{
$Stroutline = $Stroutline + "No,"
$missingfile = "Yes"
}
}
$Stroutline = $Stroutline + $missingfile
Write-Output $Stroutline | Out-File "$reportName" -Force -Append
$j++
}Following is the script i modified using threads example. this is running even slower.
[array]$contentArray = @()
[array]$allfileslist = @()
[array]$uniquefileslist = @()
[array]$allfilesSRVlist = @()
[array]$srvlist = @()
$FolderName = Split-Path -parent $MyInvocation.MyCommand.Definition
$reportName = $FolderName + "\ComparisonReport3.csv"
$ListOfFiles = get-childitem $FolderName
$List = $ListOfFiles | where {$_.extension -ieq ".txt"}
$list.count
$index = 0
#$contentArray = New-Object 'object[,]' $xDim, $yDim
foreach($listitem in $List){
$listfilename = $listitem.FullName
$listname = $listitem.Name
$listname = $listname.replace(".txt","")
$srvlist = $srvlist + $listname
write-host $listfilename
#$StrContent = get-content $listfilename
$StrContent = [io.file]::ReadAllLines($listfilename)
$contentArray += ,@($StrContent)
$StrContent.count
$contentArray[$index].count
$index = $index + 1
}
for($i = 0;$i -lt $index;$i++){
$allfileslist = $allfileslist+ $contentArray[$i]
}
$allfileslist.count
#$uniquefileslist = $allfileslist | select –unique
get-date
$uniquefileslist = $allfileslist | sort-object | get-unique
$uniquefileslist.count
get-date
$Stroutline = "File Name,"
foreach($srvlistitem in $srvlist){
$Stroutline = $Stroutline + $srvlistitem.ToUpper() + ","
}
$Stroutline = $Stroutline + "Is it Missing file?"
Write-Output $Stroutline | Out-File "$reportName" -Force
$j = 1
$count = $uniquefileslist.count
$stroutput = ""
$maxConcurrent = 50
$results= ""
$PauseTime = 1
$uniquefileslist | %{
while ((Get-Job -State Running).Count -ge $maxConcurrent) {Start-Sleep -seconds $PauseTime}
$job = start-job -argumentList $_,$contentArray,$index -scriptblock {
$StrArgFileName = $args[0]
$ArgContentArray = $args[1]
$ArgIndex = $args[2]
$Stroutline = ""
$missingfile = "No"
$Stroutline = $StrArgFileName + ","
for($i=0;$i -lt $ArgIndex;$i++){
if($ArgContentArray[$i] -contains $StrArgFileName){
$Stroutline = $Stroutline + "Yes,"
}
else{
$Stroutline = $Stroutline + "No,"
$missingfile = "Yes"
}
}
$Stroutline = $Stroutline + $missingfile
$Stroutline
}
While (Get-Job -State "Running")
{
Start-Sleep 1
}
$results = Get-Job | Receive-Job
$results
$stroutput = $stroutput + "`n" + $results
Remove-Job *
}
Write-Output $stroutput | Out-File "$reportName" -Force -Append Thank you very much!
Vamsi.