From 3046a4c983d3ccfd5e3e753a9f1321cccce3eef1 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 8 Feb 2024 10:52:43 -0500 Subject: [PATCH] Install LME in the testbed from a single script (#150) * Adding the configure scripts * Add scripts to zip and copy to a container for downloading in the server * Grab the expiry time properly in copy file * Overwrite the blob if it exists * Add the script to download file into DC * Script that unzips the files in a container * Adds username argument to download files * Add script to run scripts in container * Adds username argument to gpo script * Modifies the url name in the client GPO * Adds the functionality for chapter 1 and first half of chapter 2 * Imports the sysmon GPO * Update the variables for sysmon gpo * Name the scripts so they are grouped together in a listing * Echos the file download url * Expands the domain name correctly in create ou * Write the url output of copy file to container to a different output stream * Create a new LME folder for our scripts and files * Set path for extract to lme * Update paths for scripts to /lme * Fix the wec server name setting * Adds the scripts to install chapter 1 and 2 * Allows azure to download in linux and windows * Adds linux install scripts. * Adds winlogbeat installer * emove garbage in update server name * Tweak several scripts to get the scp of files_for_windows * Adds installer script to run all the scripts * Fixes the formatting method for az output * Clean up the scripts and add documentation * Fixes outputting format errors * Fixes hanging on adding ls1 to domain * Fix formatting errors on responses * Update linux expect script for different prompts. * Handle the reboot message for linux expect script * Echos the file download url * Create a new LME folder for our scripts and files * Set path for extract to lme * Update paths for scripts to /lme * Update paths for scripts to /lme * Fix the wec server name setting * Adds the scripts to install chapter 1 and 2 * Allows azure to download in linux and windows * Adds linux install scripts. * Adds winlogbeat installer * emove garbage in update server name * Tweak several scripts to get the scp of files_for_windows * Adds installer script to run all of the scripts * Fixes the formatting method for az output * Clean up the scripts and add documentation * Fixes outputting format errors * Fixes hanging on adding ls1 to domain * Fix formatting errors on responses * Update linux expect script for different prompts. * Handle the reboot message for linux expect script * Adds InstallTestbed instructions to Readme.md * Modifies parameters to be pascal case * ls1 not being set on DC1 * Adds Linux Only install to SetupTestbed * Remove separate linux only script * Update testing/Readme.md Co-authored-by: Alden Hilton <106177711+adhilto@users.noreply.github.com> * Make number of clients consisten between scripts * Add ports for elk stack for testing * Update readmes to change ResourceGroupName to ResourceGroup * Adds a switch to install linux only * Adds simple tests to check install * Removes the error if the old configure zip is not found. * Adds variables to linux tests run command * Move credential extraction to lib for use by other scripts. * Adds npm for other testing * Adds latest version of nodejs for testing * Make output.log readable for tests * Add the -m parameter in the testing readme * Download the latest version or a specified version * Reboot for 1.3.0 * Notes that we could have different expect scripts * Put back in the restart after all of the domain updates * Scp uses ls1 instead of ls1.lme.local * Up the timeout of the adding ls1.lme.local * Up the timeout of the adding ls1.lme.local * Fixes chmod of the output.log for tests * Adds venv to the gitignore * Adds the ability to pass a branch to the installer * Remove node installer * Change timeout in expect script for slow connections * Make shell files executable --------- Co-authored-by: Clint Baxley Co-authored-by: Alden Hilton <106177711+adhilto@users.noreply.github.com> --- .gitignore | 2 + testing/InstallTestbed.ps1 | 386 ++++++++++++++ testing/Readme.md | 67 ++- testing/SetupTestbed.ps1 | 478 +++++++++++------- .../azure_scripts/copy_file_to_container.ps1 | 81 +++ .../azure_scripts/create_blob_container.ps1 | 95 ++++ .../azure_scripts/download_in_container.ps1 | 106 ++++ .../azure_scripts/extract_archive.ps1 | 90 ++++ .../azure_scripts/lib/utilityFunctions.ps1 | 143 ++++++ .../azure_scripts/run_script_in_container.ps1 | 59 +++ .../azure_scripts/zip_my_parents_parent.ps1 | 34 ++ testing/configure/chown_dc1_private_key.ps1 | 21 + testing/configure/create_lme_directory.ps1 | 27 + testing/configure/create_ou.ps1 | 23 + testing/configure/download_files.ps1 | 23 + testing/configure/install_chapter_1.ps1 | 65 +++ testing/configure/install_chapter_2.ps1 | 28 + testing/configure/lib/functions.sh | 31 ++ .../configure/linux_authorize_private_key.sh | 4 + testing/configure/linux_install_lme.exp | 56 ++ testing/configure/linux_install_lme.sh | 85 ++++ testing/configure/linux_make_private_key.exp | 16 + testing/configure/linux_test_install.sh | 119 +++++ testing/configure/linux_update_system.sh | 3 + .../list_computers_forwarding_events.ps1 | 27 + testing/configure/move_computers_to_ou.ps1 | 38 ++ testing/configure/sysmon_gpo_update_vars.ps1 | 43 ++ testing/configure/sysmon_import_gpo.ps1 | 34 ++ .../configure/sysmon_install_in_sysvol.ps1 | 69 +++ testing/configure/sysmon_link_gpo.ps1 | 18 + testing/configure/trust_ls1_ssh_key.ps1 | 66 +++ testing/configure/wec_firewall.ps1 | 18 + .../configure/wec_gpo_update_server_name.ps1 | 42 ++ testing/configure/wec_import_gpo.ps1 | 34 ++ testing/configure/wec_link_gpo.ps1 | 27 + testing/configure/wec_service_provisioner.ps1 | 24 + testing/configure/wec_start_service.ps1 | 19 + testing/configure/winlogbeat_install.ps1 | 84 +++ 38 files changed, 2377 insertions(+), 208 deletions(-) create mode 100644 testing/InstallTestbed.ps1 create mode 100644 testing/configure/azure_scripts/copy_file_to_container.ps1 create mode 100644 testing/configure/azure_scripts/create_blob_container.ps1 create mode 100644 testing/configure/azure_scripts/download_in_container.ps1 create mode 100644 testing/configure/azure_scripts/extract_archive.ps1 create mode 100644 testing/configure/azure_scripts/lib/utilityFunctions.ps1 create mode 100644 testing/configure/azure_scripts/run_script_in_container.ps1 create mode 100644 testing/configure/azure_scripts/zip_my_parents_parent.ps1 create mode 100644 testing/configure/chown_dc1_private_key.ps1 create mode 100644 testing/configure/create_lme_directory.ps1 create mode 100644 testing/configure/create_ou.ps1 create mode 100644 testing/configure/download_files.ps1 create mode 100644 testing/configure/install_chapter_1.ps1 create mode 100644 testing/configure/install_chapter_2.ps1 create mode 100644 testing/configure/lib/functions.sh create mode 100755 testing/configure/linux_authorize_private_key.sh create mode 100755 testing/configure/linux_install_lme.exp create mode 100755 testing/configure/linux_install_lme.sh create mode 100755 testing/configure/linux_make_private_key.exp create mode 100755 testing/configure/linux_test_install.sh create mode 100755 testing/configure/linux_update_system.sh create mode 100644 testing/configure/list_computers_forwarding_events.ps1 create mode 100644 testing/configure/move_computers_to_ou.ps1 create mode 100644 testing/configure/sysmon_gpo_update_vars.ps1 create mode 100644 testing/configure/sysmon_import_gpo.ps1 create mode 100644 testing/configure/sysmon_install_in_sysvol.ps1 create mode 100644 testing/configure/sysmon_link_gpo.ps1 create mode 100644 testing/configure/trust_ls1_ssh_key.ps1 create mode 100644 testing/configure/wec_firewall.ps1 create mode 100644 testing/configure/wec_gpo_update_server_name.ps1 create mode 100644 testing/configure/wec_import_gpo.ps1 create mode 100644 testing/configure/wec_link_gpo.ps1 create mode 100644 testing/configure/wec_service_provisioner.ps1 create mode 100644 testing/configure/wec_start_service.ps1 create mode 100644 testing/configure/winlogbeat_install.ps1 diff --git a/.gitignore b/.gitignore index 5b650322..a3dc61a8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ dashboard_update.sh files_for_windows.zip lme.conf lme_update.sh +/testing/tests/.env +/testing/tests/venv/ diff --git a/testing/InstallTestbed.ps1 b/testing/InstallTestbed.ps1 new file mode 100644 index 00000000..9d297be5 --- /dev/null +++ b/testing/InstallTestbed.ps1 @@ -0,0 +1,386 @@ +param ( + [Alias("g")] + [Parameter(Mandatory = $true)] + [string]$ResourceGroup, + + [Alias("w")] + [string]$DomainController = "DC1", + + [Alias("l")] + [string]$LinuxVM = "LS1", + + [Alias("n")] + [int]$NumClients = 2, + + [Alias("m")] + [Parameter( + HelpMessage = "(minimal) Only install the linux server. Useful for testing the linux server without the windows clients" + )] + [switch]$LinuxOnly, + + [Alias("v")] + [string]$Version = $false, + + [Alias("b")] + [string]$Branch = $false +) + +# If you were to need the password from the SetupTestbed.ps1 script, you could use this: +# $Password = Get-Content "${ResourceGroup}.password.txt" + +$ProcessSeparator = "`n----------------------------------------`n" + +# Define our library path +$LibraryPath = Join-Path -Path $PSScriptRoot -ChildPath "configure\azure_scripts\lib\utilityFunctions.ps1" + +# Check if the library file exists +if (Test-Path -Path $LibraryPath) { + # Dot-source the library script + . $LibraryPath +} +else { + Write-Error "Library script not found at path: $LibraryPath" +} + +if ($Version -ne $false -and -not ($Version -match '^[0-9]+\.[0-9]+\.[0-9]+$')) { + Write-Host "Invalid version format: $Version. Expected format: X.Y.Z (e.g., 1.3.0)" + exit 1 +} + +# Create a container to keep files for the VM +Write-Output "Creating a container to keep files for the VM..." +$createBlobResponse = ./configure/azure_scripts/create_blob_container.ps1 ` + -ResourceGroup $ResourceGroup +Write-Output $createBlobResponse +Write-Output $ProcessSeparator + +# Source the variables from the file +Write-Output "`nSourcing the variables from the file..." +. ./configure/azure_scripts/config.ps1 + +# Remove old code if it exists +if (Test-Path ./configure.zip) { + Remove-Item ./configure.zip -Force -Confirm:$false -ErrorAction SilentlyContinue +} + +Write-Output $ProcessSeparator + +# Zip up the installer scripts for the VM +Write-Output "`nZipping up the installer scripts for the VMs..." +./configure/azure_scripts/zip_my_parents_parent.ps1 +Write-Output $ProcessSeparator + +# Upload the zip file to the container and get a key to download it +Write-Output "`nUploading the zip file to the container and getting a key to download it..." +$FileDownloadUrl = ./configure/azure_scripts/copy_file_to_container.ps1 ` + -LocalFilePath "configure.zip" ` + -ContainerName $ContainerName ` + -StorageAccountName $StorageAccountName ` + -StorageAccountKey $StorageAccountKey + +Write-Output "File download URL: $FileDownloadUrl" +Write-Output $ProcessSeparator + +Write-Output "`nChanging directory to the azure scripts..." +Set-Location configure/azure_scripts +Write-Output $ProcessSeparator + +if (-Not $LinuxOnly) { + Write-Output "`nInstalling on the windows clients..." + # Make our directory on the VM + Write-Output "`nMaking our directory on the VM..." + $createDirResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --name $DomainController ` + --resource-group $ResourceGroup ` + --scripts "if (-not (Test-Path -Path 'C:\lme')) { New-Item -Path 'C:\lme' -ItemType Directory }" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$createDirResponse") + Write-Output $ProcessSeparator + + # Download the zip file to the VM + Write-Output "`nDownloading the zip file to the VM..." + $downloadZipFileResponse = .\download_in_container.ps1 ` + -VMName $DomainController ` + -ResourceGroup $ResourceGroup ` + -FileDownloadUrl "$FileDownloadUrl" ` + -DestinationFilePath "configure.zip" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$downloadZipFileResponse") + Write-Output $ProcessSeparator + + # Extract the zip file + Write-Output "`nExtracting the zip file..." + $extractArchiveResponse = .\extract_archive.ps1 ` + -VMName $DomainController ` + -ResourceGroup $ResourceGroup ` + -FileName "configure.zip" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$extractArchiveResponse") + Write-Output $ProcessSeparator + + # Run the install script for chapter 1 + Write-Output "`nRunning the install script for chapter 1..." + $installChapter1Response = .\run_script_in_container.ps1 ` + -ResourceGroup $ResourceGroup ` + -VMName $DomainController ` + -ScriptPathOnVM "C:\lme\configure\install_chapter_1.ps1" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installChapter1Response") + Write-Output $ProcessSeparator + + # Update the group policy on the remote machines + Write-Output "`nUpdating the group policy on the remote machines..." + Invoke-GPUpdateOnVMs -ResourceGroup $ResourceGroup -numberOfClients $NumClients + Write-Output $ProcessSeparator + + # Wait for the services to start + Write-Output "`nWaiting for the services to start..." + Start-Sleep 10 + + # See if we can see the forwarding computers in the DC + write-host "`nChecking if we can see the forwarding computers in the DC..." + $listForwardingComputersResponse = .\run_script_in_container.ps1 ` + -ResourceGroup $ResourceGroup ` + -VMName $DomainController ` + -ScriptPathOnVM "C:\lme\configure\list_computers_forwarding_events.ps1" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$listForwardingComputersResponse") + Write-Output $ProcessSeparator + + # Install the sysmon service on DC1 from chapter 2 + Write-Output "`nInstalling the sysmon service on DC1 from chapter 2..." + $installChapter2Response = .\run_script_in_container.ps1 ` + -ResourceGroup $ResourceGroup ` + -VMName $DomainController ` + -ScriptPathOnVM "C:\lme\configure\install_chapter_2.ps1" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installChapter2Response") + Write-Output $ProcessSeparator + + # Update the group policy on the remote machines + Write-Output "`nUpdating the group policy on the remote machines..." + Invoke-GPUpdateOnVMs -ResourceGroup $ResourceGroup -numberOfClients $NumClients + Write-Output $ProcessSeparator + + # Wait for the services to start + Write-Output "`nWaiting for the services to start. Generally they don't show..." + Start-Sleep 10 + + # See if you can see sysmon running on the machine + Write-Output "`nSeeing if you can see sysmon running on a machine..." + $showSysmonResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --name "C1" ` + --resource-group $ResourceGroup ` + --scripts 'Get-Service | Where-Object { $_.DisplayName -like "*Sysmon*" }' + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$showSysmonResponse") + Write-Output $ProcessSeparator +} + +Write-Output "`nInstalling on the linux server..." +# Download the installers on LS1 +Write-Output "`nDownloading the installers on LS1..." +$downloadLinuxZipFileResponse = .\download_in_container.ps1 ` + -VMName $LinuxVM ` + -ResourceGroup $ResourceGroup ` + -FileDownloadUrl "$FileDownloadUrl" ` + -DestinationFilePath "configure.zip" ` + -os "linux" +Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$downloadLinuxZipFileResponse") +Write-Output $ProcessSeparator + +# Install unzip on LS1 +Write-Output "`nInstalling unzip on LS1..." +$installUnzipResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts 'apt-get install unzip -y' +Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installUnzipResponse") +Write-Output $ProcessSeparator + +# Unzip the file on LS1 +Write-Output "`nUnzipping the file on LS1..." +$extractLinuxArchiveResponse = .\extract_archive.ps1 ` + -VMName $LinuxVM ` + -ResourceGroup $ResourceGroup ` + -FileName "configure.zip" ` + -Os "Linux" +Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$extractLinuxArchiveResponse") +Write-Output $ProcessSeparator + +Write-Output "`nMaking the installer files executable and updating the system packages on LS1..." +$updateLinuxResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts 'chmod +x /home/admin.ackbar/lme/configure/* && /home/admin.ackbar/lme/configure/linux_update_system.sh' +Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$updateLinuxResponse") +Write-Output $ProcessSeparator + +$versionArgument = "" +if ($Branch -ne $false) { + $versionArgument = " -b '$($Branch)'" +} elseif ($Version -ne $false) { + $versionArgument = " -v $Version" +} +Write-Output "`nRunning the lme installer on LS1..." +$installLmeResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts "/home/admin.ackbar/lme/configure/linux_install_lme.sh $versionArgument" +Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installLmeResponse") +Write-Output $ProcessSeparator + +# Check if the response contains the need to reboot +$rebootCheckstring = $installLmeResponse | Out-String +if ($rebootCheckstring -match "reboot is required in order to proceed with the install") { + # Have to check for the reboot thing here + Write-Output "`nRebooting ${LinuxVM}..." + az vm restart ` + --resource-group $ResourceGroup ` + --name $LinuxVM + Write-Output $ProcessSeparator + + Write-Output "`nRunning the lme installer on LS1..." + $installLmeResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts "/home/admin.ackbar/lme/configure/linux_install_lme.sh $versionArgument" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installLmeResponse") + Write-Output $ProcessSeparator +} + +# Capture the output of the install script +Write-Output "`nCapturing the output of the install script for ES passwords..." +$getElasticsearchPasswordsResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts 'tail -n14 "/opt/lme/Chapter 3 Files/output.log" | head -n9' + +# Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$getElasticsearchPasswordsResponse") +Write-Output $ProcessSeparator + +if (-Not $LinuxOnly){ + # Generate key using expect on linux + Write-Output "`nGenerating key using expect on linux..." + $generateKeyResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts '/home/admin.ackbar/lme/configure/linux_make_private_key.exp' + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$generateKeyResponse") + Write-Output $ProcessSeparator + + # Add the public key to the authorized_keys file on LS1 + Write-Output "`nAdding the public key to the authorized_keys file on LS1..." + $authorizePrivateKeyResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts '/home/admin.ackbar/lme/configure/linux_authorize_private_key.sh' + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$authorizePrivateKeyResponse") + Write-Output $ProcessSeparator + + # Cat the private key and capture that to the azure shell + Write-Output "`nCat the private key and capture that to the azure shell..." + $jsonResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts 'cat /home/admin.ackbar/.ssh/id_rsa' + $privateKey = Get-PrivateKeyFromJson -jsonResponse "$jsonResponse" + + # Save the private key to a file + Write-Output "`nSaving the private key to a file..." + $privateKeyPath = ".\id_rsa" + Set-Content -Path $privateKeyPath -Value $privateKey + Write-Output $ProcessSeparator + + # Upload the private key to the container and get a key to download it + Write-Output "`nUploading the private key to the container and getting a key to download it..." + $KeyDownloadUrl = ./copy_file_to_container.ps1 ` + -LocalFilePath "id_rsa" ` + -ContainerName $ContainerName ` + -StorageAccountName $StorageAccountName ` + -StorageAccountKey $StorageAccountKey + + # Download the private key to DC1 + Write-Output "`nDownloading the private key to DC1..." + $downloadPrivateKeyResponse = .\download_in_container.ps1 ` + -VMName $DomainController ` + -ResourceGroup $ResourceGroup ` + -FileDownloadUrl "$KeyDownloadUrl" ` + -DestinationFilePath "id_rsa" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$downloadPrivateKeyResponse") + Write-Output $ProcessSeparator + + # Change the ownership of the private key file on DC1 + Write-Output "`nChanging the ownership of the private key file on DC1..." + $chownPrivateKeyResponse = .\run_script_in_container.ps1 ` + -ResourceGroup $ResourceGroup ` + -VMName $DomainController ` + -ScriptPathOnVM "C:\lme\configure\chown_dc1_private_key.ps1" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$chownPrivateKeyResponse") + Write-Output $ProcessSeparator + + # Remove the private key from the local machine + Remove-Item -Path $privateKeyPath + + # Use the azure shell to run scp on DC1 to copy the files from LS1 to DC1 + Write-Output "`nUsing the azure shell to run scp on DC1 to copy the files from LS1 to DC1..." + $scpResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --name $DomainController ` + --resource-group $ResourceGroup ` + --scripts 'scp -o StrictHostKeyChecking=no -i "C:\lme\id_rsa" admin.ackbar@ls1:/home/admin.ackbar/files_for_windows.zip "C:\lme\"' + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$scpResponse") + Write-Output $ProcessSeparator + + # Extract the files on DC1 + Write-Output "`nExtracting the files on DC1..." + $extractFilesForWindowsResponse = .\extract_archive.ps1 ` + -VMName $DomainController ` + -ResourceGroup $ResourceGroup ` + -FileName "files_for_windows.zip" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$extractFilesForWindowsResponse") + Write-Output $ProcessSeparator + + # Install winlogbeat on DC1 + Write-Output "`nInstalling winlogbeat on DC1..." + $installWinlogbeatResponse = .\run_script_in_container.ps1 ` + -ResourceGroup $ResourceGroup ` + -VMName $DomainController ` + -ScriptPathOnVM "C:\lme\configure\winlogbeat_install.ps1" + + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installWinlogbeatResponse") + Write-Output $ProcessSeparator +} + + +Write-Output "`nRunning the tests for lme on LS1..." +$runTestResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --name $LinuxVM ` + --resource-group $ResourceGroup ` + --scripts '/home/admin.ackbar/lme/configure/linux_test_install.sh' | ConvertFrom-Json + +$message = $runTestResponse.value[0].message +Write-Host "$message`n" +Write-Host "--------------------------------------------" + +# Check if there is stderr content in the message field +if ($message -match '\[stderr\]\n(.+)$') { + Write-Host "Tests failed" + exit 1 +} else { + Write-Host "Tests succeeded" +} + +Write-Output "`nInstall completed." + +$EsPasswords = (Format-AzVmRunCommandOutput -JsonResponse "$getElasticsearchPasswordsResponse")[0].StdOut +# Output the passwords +$EsPasswords + +# Write the passwords to a file +$PasswordPath = "..\..\${ResourceGroup}.password.txt" +$EsPasswords | Out-File -Append -FilePath $PasswordPath \ No newline at end of file diff --git a/testing/Readme.md b/testing/Readme.md index 45301981..8577bf09 100644 --- a/testing/Readme.md +++ b/testing/Readme.md @@ -13,14 +13,16 @@ Using the Azure CLI, it creates the following: This script does not install LME; it simply creates a fresh environment that's ready to have LME installed. ## Usage -| **Parameter** | **Alias** | **Description** | **Required** | -|------------------------|-----------|----------------------------------------------------------------------------------------|---------------------------------------| -| $ResourceGroup | -g | The name of the resource group that will be created for storing all testbed resources. | Yes | -| $NumClients | -n | The number of Windows clients to create; maximum 16; defaults to 1 | No | -| $AutoShutdownTime | | The auto-shutdown time in UTC (HHMM, e.g. 2230, 0000, 1900); auto-shutdown not configured if not provided | No | -| $AutoShutdownEmail | | An email to be notified if a VM is auto-shutdown. | No | -| $AllowedSources | -s | Comma-Separated list of CIDR prefixes or IP ranges, e.g. XX.XX.XX.XX/YY,XX.XX.XX.XX/YY,etc..., that are allowed to connect to the VMs via RDP and ssh. | Yes | -| $NoPrompt | -y | Switch, run the script with no prompt (useful for automated runs). By default, the script will prompt the user to review paramters and confirm before continuing. | No | +| **Parameter** | **Alias** | **Description** | **Required** | +|--------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| +| $ResourceGroup | -g | The name of the resource group that will be created for storing all testbed resources. | Yes | +| $NumClients | -n | The number of Windows clients to create; maximum 16; defaults to 2 | No | +| $AutoShutdownTime | | The auto-shutdown time in UTC (HHMM, e.g. 2230, 0000, 1900); auto-shutdown not configured if not provided | No | +| $AutoShutdownEmail | | An email to be notified if a VM is auto-shutdown. | No | +| $AllowedSources | -s | Comma-Separated list of CIDR prefixes or IP ranges, e.g. XX.XX.XX.XX/YY,XX.XX.XX.XX/YY,etc..., that are allowed to connect to the VMs via RDP and ssh. | Yes | +| $Location | -l | The region you would like to build the assets in. Defaults to westus | No | +| $NoPrompt | -y | Switch, run the script with no prompt (useful for automated runs). By default, the script will prompt the user to review paramters and confirm before continuing. | No | +| $LinuxOnly | -m | Run a minimal install of only the linux server | No | Example: ``` @@ -28,14 +30,14 @@ Example: ``` ## Running Using Azure Shell -| **#** | **Step** | **Screenshot** | -|-------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------| -| 1 | Open a cloud shell by navigating to portal.azure.com and clicking the shell icon. | ![image](/docs/imgs/testing-screenshots/shell.png) | -| 2 | Select PowerShell. | ![image](/docs/imgs/testing-secreenshots/shell2.png) | -| 3 | Upload `SetupTestbed.ps1` by clicking the "Upload/Download files" icon | ![image](/docs/imgs/testing-screenshots/shell3.png) | -| 4 | Run the script, providing values for the parameters when promoted (see [Usage](#usage)). The script will take ~20 minutes to run to completion. | ![image](/docs/imgs/testing-screenshots/shell4.png) | -| 5 | Save the login credentials printed to the terminal at the end. At this point you can login to each VM using RDP (for the Windows servers) or SSH (for the Linux server). | ![image](/docs/imgs/testing-screenshots/shell5.png) | -| 6 | When you're done testing, simply delete the resource group to clean up all resources created. | ![image](/docs/imgs/testing-screenshots/delete.png) | +| **#** | **Step** | **Screenshot** | +|-------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| 1 | Open a cloud shell by navigating to portal.azure.com and clicking the shell icon. | ![image](/docs/imgs/testing-screenshots/shell.png) | +| 2 | Select PowerShell. | ![image](/docs/imgs/testing-secreenshots/shell2.png) | +| 3 | Clone the repo `git clone https://github.com/cisagov/LME.git` and then `cd LME\testing` | | +| 4 | Run the script, providing values for the parameters when promoted (see [Usage](#usage)). The script will take ~20 minutes to run to completion. | ![image](/docs/imgs/testing-screenshots/shell4.png) | +| 5 | Save the login credentials printed to the terminal at the end (They will also be in a file called `<$ResourceGroup>.password.txt`). At this point you can login to each VM using RDP (for the Windows servers) or SSH (for the Linux server). | ![image](/docs/imgs/testing-screenshots/shell5.png) | +| 6 | When you're done testing, simply delete the resource group to clean up all resources created. | ![image](/docs/imgs/testing-screenshots/delete.png) | # Extra Functionality: @@ -55,3 +57,36 @@ Flags: - enable: deletes the DENYINTERNET/DENYLOADBALANCER rules - NSG: sets NSG to a custom NSG if desired [NSG1 default] +## Install LME on the cluster: +### InstallTestbed.ps1 +## Usage +| **Parameter** | **Alias** | **Description** | **Required** | +|-------------------|-----------|----------------------------------------------------------------------------------------|--------------| +| $ResourceGroup | -g | The name of the resource group that will be created for storing all testbed resources. | Yes | +| $NumClients | -n | The number of Windows clients you have created; defaults to 2 | No | +| $DomainController | -w | The name of the domain controller in the cluster; defaults to "DC1" | No | +| $LinuxVm | -l | The name of the linux server in the cluster; defaults to "LS1" | No | +| $LinuxOnly | -m | Run a minimal install of only the linux server | No | +| $Version | -v | Optionally provide a version to install if you want a specific one. `-v 1.3.2` | No | +| $Branch | -b | Optionally provide a branch to install if you want a specific one `-b your_branch` | No | + +Example: +``` +./InstallTestbed.ps1 -ResourceGroup YourResourceGroup +# Or if you want to save the output to a file +./InstallTestbed.ps1 -ResourceGroup YourResourceGroup | Tee-Object -FilePath "./YourResourceGroup.output.log" +``` +| **#** | **Step** | **Screenshot** | +|-------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------| +| 1 | Open a cloud shell by navigating to portal.azure.com and clicking the shell icon. | ![image](/docs/imgs/testing-screenshots/shell.png) | +| 2 | Select PowerShell. | ![image](/docs/imgs/testing-secreenshots/shell2.png) | +| 3.a | If you have already cloned the LME repo then make sure you are in the `LME\testing` directory and run git pull before changing to the testing directory. | | +| 3.b | If you haven't cloned it, clone the github repo in the home directory. `git clone https://github.com/cisagov/LME.git` and then `cd LME\testing`. | | +| 4 | Now you can run one of the commands from the Examples above. | | +| 5 | Save the login credentials printed to the terminal at the end. *See note* | | +| 6 | When you're done testing, simply delete the resource group to clean up all resources created. | | + +Note: When the script finishes you will be in the azure_scripts directory, and you should see the elasticsearch credentials printed to the terminal. +You will need to `cd ../../` to get back to the LME directory. All the passwords should also be in the `<$ResourceGroup>.password.txt` file. + + diff --git a/testing/SetupTestbed.ps1 b/testing/SetupTestbed.ps1 index 4c5a347b..6e189f99 100644 --- a/testing/SetupTestbed.ps1 +++ b/testing/SetupTestbed.ps1 @@ -4,7 +4,7 @@ Creates the following: - A resource group - A virtual network, subnet, and network security group - - 2 VMs: "DC1," a Windows server, and "LS1," a Linux server + - 2 VMs: "DC1," a Windows server, and "LS1," a Linux server. You can use -m for only the linux server - Client VMs: Windows clients "C1", "C2", etc. up to 16 based on user input - Promotes DC1 to a domain controller - Adds "C" clients to the managed domain @@ -18,45 +18,66 @@ #> param ( - [Parameter( - HelpMessage="Auto-Shutdown time in UTC (HHMM, e.g. 2230, 0000, 1900). Convert timezone as necesary: (e.g. 05:30 pm ET -> 9:30 pm UTC -> 21:30 -> 2130)" - )] - $AutoShutdownTime=$null, - - [Parameter( - HelpMessage="Auto-shutdown notification email" - )] - $AutoShutdownEmail=$null, - - [Alias("l")] - [Parameter( - HelpMessage="Location where the cluster will be built. Default westus" - )] - [string]$Location="westus", - - [Alias("g")] - [Parameter(Mandatory=$true)] - [string]$ResourceGroup, - - [Alias("n")] - [Parameter( - HelpMessage="Number of clients to create (Max: 16)" - )] - [int]$NumClients=1, - - [Alias("s")] - [Parameter(Mandatory=$true, - HelpMessage="XX.XX.XX.XX/YY,XX.XX.XX.XX/YY,etc... Comma-Separated list of CIDR prefixes or IP ranges" - )] - [string]$AllowedSources, - - [Alias("y")] - [Parameter( - HelpMessage="Run the script with no prompt (useful for automated runs)" - )] - [switch]$NoPrompt + [Parameter( + HelpMessage = "Auto-Shutdown time in UTC (HHMM, e.g. 2230, 0000, 1900). Convert timezone as necesary: (e.g. 05:30 pm ET -> 9:30 pm UTC -> 21:30 -> 2130)" + )] + $AutoShutdownTime = $null, + + [Parameter( + HelpMessage = "Auto-shutdown notification email" + )] + $AutoShutdownEmail = $null, + + [Alias("l")] + [Parameter( + HelpMessage = "Location where the cluster will be built. Default westus" + )] + [string]$Location = "westus", + + [Alias("g")] + [Parameter(Mandatory = $true)] + [string]$ResourceGroup, + + [Alias("n")] + [Parameter( + HelpMessage = "Number of clients to create (Max: 16)" + )] + [int]$NumClients = 2, + + [Alias("s")] + [Parameter(Mandatory = $true, + HelpMessage = "XX.XX.XX.XX/YY,XX.XX.XX.XX/YY,etc... Comma-Separated list of CIDR prefixes or IP ranges" + )] + [string]$AllowedSources, + + [Alias("y")] + [Parameter( + HelpMessage = "Run the script with no prompt (useful for automated runs)" + )] + [switch]$NoPrompt, + + [Alias("m")] + [Parameter( + HelpMessage = "(minimal) Only install the linux server. Useful for testing the linux server without the windows clients" + )] + [switch]$LinuxOnly ) +$ProcessSeparator = "`n----------------------------------------`n" + +# Define our library path +$libraryPath = Join-Path -Path $PSScriptRoot -ChildPath "configure\azure_scripts\lib\utilityFunctions.ps1" + +# Check if the library file exists +if (Test-Path -Path $libraryPath) { + # Dot-source the library script + . $libraryPath +} +else { + Write-Error "Library script not found at path: $libraryPath" +} + + #DEFAULTS: #Desired Netowrk Mapping: $VNetPrefix = "10.1.0.0/16" @@ -72,9 +93,9 @@ $VMAdmin = "admin.ackbar" $DomainName = "lme.local" #Port options: https://learn.microsoft.com/en-us/cli/azure/network/nsg/rule?view=azure-cli-latest#az-network-nsg-rule-create -$Ports = 22,3389 -$Priorities = 1001,1002 -$Protocols = "Tcp","Tcp" +$Ports = 22, 3389, 443, 9200, 5044 +$Priorities = 1001, 1002, 1003, 1004, 1005 +$Protocols = "Tcp", "Tcp", "Tcp", "Tcp", "Tcp" function Get-RandomPassword { @@ -105,53 +126,56 @@ function Set-AutoShutdown { Write-Output "`nCreating Auto-Shutdown Rule for $VMName at time $AutoShutdownTime..." if ($null -ne $AutoShutdownEmail) { - az vm auto-shutdown ` - -g $ResourceGroup ` - -n $VMName ` - --time $AutoShutdownTime ` - --email $AutoShutdownEmail + $autoShutdownResponse = az vm auto-shutdown ` + -g $ResourceGroup ` + -n $VMName ` + --time $AutoShutdownTime ` + --email $AutoShutdownEmail + Write-Output $autoShutdownResponse } else { - az vm auto-shutdown ` - -g $ResourceGroup ` - -n $VMName ` - --time $AutoShutdownTime + $autoShutdownResponse = az vm auto-shutdown ` + -g $ResourceGroup ` + -n $VMName ` + --time $AutoShutdownTime + Write-Output $autoShutdownResponse } } function Set-NetworkRules { - param ( - [Parameter(Mandatory)] - $AllowedSourcesList - ) - - if ($Ports.length -ne $Priorities.length){ - Write-Output "Priorities and Ports length should be equal!" - exit -1 - } - if ($Ports.length -ne $Protocols.length){ - Write-Output "Protocols and Ports length should be equal!" - exit -1 - } - - for ($i = 0; $i -le $Ports.length - 1 ; $i++) { - $port=$Ports[$i] - $priority=$Priorities[$i] - $protocol=$Protocols[$i] - Write-Output "`nCreating Network Port $port rule..." - - az network nsg rule create --name Network_Port_Rule_$port ` - --resource-group $ResourceGroup ` - --nsg-name NSG1 ` - --priority $priority ` - --direction Inbound ` - --access Allow ` - --protocol $protocol ` - --source-address-prefixes $AllowedSourcesList ` - --destination-address-prefixes '*' ` - --destination-port-ranges $port ` - --description "Allow inbound from $sources on $port via $protocol connections." - } + param ( + [Parameter(Mandatory)] + $AllowedSourcesList + ) + + if ($Ports.length -ne $Priorities.length) { + Write-Output "Priorities and Ports length should be equal!" + Exit 1 + } + if ($Ports.length -ne $Protocols.length) { + Write-Output "Protocols and Ports length should be equal!" + Exit 1 + } + + for ($i = 0; $i -le $Ports.length - 1; $i++) { + $port = $Ports[$i] + $priority = $Priorities[$i] + $protocol = $Protocols[$i] + Write-Output "`nCreating Network Port $port rule..." + + $networkRuleResponse = az network nsg rule create --name Network_Port_Rule_$port ` + --resource-group $ResourceGroup ` + --nsg-name NSG1 ` + --priority $priority ` + --direction Inbound ` + --access Allow ` + --protocol $protocol ` + --source-address-prefixes $AllowedSourcesList ` + --destination-address-prefixes '*' ` + --destination-port-ranges $port ` + --description "Allow inbound from $sources on $port via $protocol connections." + Write-Output $networkRuleResponse + } } @@ -159,23 +183,23 @@ function Set-NetworkRules { # Validation of Globals # ######################## $AllowedSourcesList = $AllowedSources -Split "," -if ($AllowedSourcesList.length -lt 1){ - Write-Output "**ERROR**: Variable AllowedSources must be set (set with -AllowedSources or -s)" - exit -1 +if ($AllowedSourcesList.length -lt 1) { + Write-Output "**ERROR**: Variable AllowedSources must be set (set with -AllowedSources or -s)" + Exit 1 } if ($null -ne $AutoShutdownTime) { - if ( -not ( $AutoShutdownTime -match '^([01][0-9]|2[0-3])[0-5][0-9]$' ) ){ + if (-not ( $AutoShutdownTime -match '^([01][0-9]|2[0-3])[0-5][0-9]$')) { Write-Output "**ERROR** Invalid time" Write-Output "Enter the Auto-Shutdown time in UTC (HHMM, e.g. 2230, 0000, 1900), `n`tConvert timezone as necesary: (e.g. 05:30 pm ET -> 9:30 pm UTC -> 21:30 -> 2130)" - exit -1 - } + Exit 1 + } } -if ($NumClients -lt 1 -or $NumClients -gt 16) { - Write-Output "The number of clients must be at least 1 and no more than 16." +if (($NumClients -lt 1 -or $NumClients -gt 16) -and -Not $LinuxOnly) { + Write-Output "The number of clients must be at least 1 and no more than 16." $NumClients = $NumClients -as [int] - exit -1 + Exit 1 } ################ @@ -189,39 +213,45 @@ Write-Output "Number of clients: $NumClients" Write-Output "Allowed sources (IP's): $AllowedSourcesList" Write-Output "Auto-shutdown time: $AutoShutdownTime" Write-Output "Auto-shutdown e-mail: $AutoShutdownEmail" +if ($LinuxOnly) { + Write-Output "Creating a linux server only" +} if (-Not $NoPrompt) { - do { - $Proceed = Read-Host "`nProceed? (Y/n)" - } until ($Proceed -eq "y" -or $Proceed -eq "Y" -or $Proceed -eq "n" -or $Proceed -eq "N") - - if ($Proceed -eq "n" -or $Proceed -eq "N") { - Write-Output "Setup canceled" - exit - } + do { + $Proceed = Read-Host "`nProceed? (Y/n)" + } until ($Proceed -eq "y" -or $Proceed -eq "Y" -or $Proceed -eq "n" -or $Proceed -eq "N") + + if ($Proceed -eq "n" -or $Proceed -eq "N") { + Write-Output "Setup canceled" + Exit + } } ######################## # Setup resource group # ######################## Write-Output "`nCreating resource group..." -az group create --name $ResourceGroup --location $Location +$createResourceGroupResponse = az group create --name $ResourceGroup --location $Location +Write-Output $createResourceGroupResponse ################# # Setup network # ################# Write-Output "`nCreating virtual network..." -az network vnet create --resource-group $ResourceGroup ` +$createVirtualNetworkResponse = az network vnet create --resource-group $ResourceGroup ` --name VNet1 ` --address-prefix $VNetPrefix ` --subnet-name SNet1 ` --subnet-prefix $SubnetPrefix +Write-Output $createVirtualNetworkResponse Write-Output "`nCreating nsg..." -az network nsg create --name NSG1 ` +$createNsgResponse = az network nsg create --name NSG1 ` --resource-group $ResourceGroup ` --location $Location +Write-Output $createNsgResponse Set-NetworkRules -AllowedSourcesList $AllowedSourcesList @@ -229,24 +259,12 @@ Set-NetworkRules -AllowedSourcesList $AllowedSourcesList # Create the VMs # ################## $VMPassword = Get-RandomPassword 12 -Write-Output "`nWriting $VMAdmin password to password.txt" -echo $VMPassword > password.txt +Write-Output "`nWriting $VMAdmin password to ${ResourceGroup}.password.txt" +$VMPassword | Out-File -FilePath "${ResourceGroup}.password.txt" -Encoding UTF8 -Write-Output "`nCreating DC1..." -az vm create ` - --name DC1 ` - --resource-group $ResourceGroup ` - --nsg NSG1 ` - --image Win2019Datacenter ` - --admin-username $VMAdmin ` - --admin-password $VMPassword ` - --vnet-name VNet1 ` - --subnet SNet1 ` - --public-ip-sku Standard ` - --private-ip-address $DcIP Write-Output "`nCreating LS1..." -az vm create ` +$createLs1Response = az vm create ` --name LS1 ` --resource-group $ResourceGroup ` --nsg NSG1 ` @@ -259,11 +277,12 @@ az vm create ` --size Standard_E2d_v4 ` --os-disk-size-gb 128 ` --private-ip-address $LsIP +Write-Output $createLs1Response -for ($i = 1; $i -le $NumClients; $i++) { - Write-Output "`nCreating C$i..." - az vm create ` - --name C$i ` +if (-Not $LinuxOnly){ + Write-Output "`nCreating DC1..." + $createDc1Response = az vm create ` + --name DC1 ` --resource-group $ResourceGroup ` --nsg NSG1 ` --image Win2019Datacenter ` @@ -271,7 +290,23 @@ for ($i = 1; $i -le $NumClients; $i++) { --admin-password $VMPassword ` --vnet-name VNet1 ` --subnet SNet1 ` - --public-ip-sku Standard + --public-ip-sku Standard ` + --private-ip-address $DcIP + Write-Output $createDc1Response + for ($i = 1; $i -le $NumClients; $i++) { + Write-Output "`nCreating C$i..." + $createClientResponse = az vm create ` + --name C$i ` + --resource-group $ResourceGroup ` + --nsg NSG1 ` + --image Win2019Datacenter ` + --admin-username $VMAdmin ` + --admin-password $VMPassword ` + --vnet-name VNet1 ` + --subnet SNet1 ` + --public-ip-sku Standard + Write-Output $createClientResponse + } } ########################### @@ -279,100 +314,159 @@ for ($i = 1; $i -le $NumClients; $i++) { ########################### if ($null -ne $AutoShutdownTime) { - Set-AutoShutdown "DC1" Set-AutoShutdown "LS1" - for ($i = 1; $i -le $NumClients; $i++) { - Set-AutoShutdown "C$i" + if (-Not $LinuxOnly){ + Set-AutoShutdown "DC1" + for ($i = 1; $i -le $NumClients; $i++) { + Set-AutoShutdown "C$i" + } } } #################### # Setup the domain # #################### -Write-Output "`nInstalling AD Domain services on DC1..." -az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name DC1 ` - --scripts "Add-WindowsFeature AD-Domain-Services -IncludeManagementTools" +if (-Not $LinuxOnly){ + Write-Output "`nInstalling AD Domain services on DC1..." + $addDomainServicesResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name DC1 ` + --scripts "Add-WindowsFeature AD-Domain-Services -IncludeManagementTools" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addDomainServicesResponse") -Write-Output "`nRestarting DC1..." -az vm restart ` - --resource-group $ResourceGroup ` - --name DC1 ` +# Write-Output "`nRestarting DC1..." +# az vm restart ` +# --resource-group $ResourceGroup ` +# --name DC1 ` -Write-Output "`nCreating the ADDS forest..." -az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name DC1 ` - --scripts "`$Password = ConvertTo-SecureString `"$VMPassword`" -AsPlainText -Force; ` -Install-ADDSForest -DomainName $DomainName -Force -SafeModeAdministratorPassword `$Password" + Write-Output "`nCreating the ADDS forest..." + $installAddsForestResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name DC1 ` + --scripts "`$Password = ConvertTo-SecureString `"$VMPassword`" -AsPlainText -Force; ` + Install-ADDSForest -DomainName $DomainName -Force -SafeModeAdministratorPassword `$Password" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installAddsForestResponse") -Write-Output "`nRestarting DC1..." -az vm restart ` - --resource-group $ResourceGroup ` - --name DC1 ` + Write-Output "`nRestarting DC1..." + az vm restart ` + --resource-group $ResourceGroup ` + --name DC1 ` -for ($i = 1; $i -le $NumClients; $i++) { - Write-Output "`nAdding DC IP address to C$i host file..." - az vm run-command invoke ` - --command-id RunPowerShellScript ` + for ($i = 1; $i -le $NumClients; $i++) { + Write-Output "`nAdding DC IP address to C$i host file..." + $addIpResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name C$i ` + --scripts "Add-Content -Path `$env:windir\System32\drivers\etc\hosts -Value `"`n$DcIP`t$DomainName`" -Force" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addIpResponse") + + Write-Output "`nSetting C$i DNS server to DC1..." + $setDnsResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name C$i ` + --scripts "Get-Netadapter | Set-DnsClientServerAddress -ServerAddresses $DcIP" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$setDnsResponse") + + Write-Output "`nRestarting C$i..." + az vm restart ` --resource-group $ResourceGroup ` --name C$i ` - --scripts "Add-Content -Path `$env:windir\System32\drivers\etc\hosts -Value `"`n$DcIP`t$DomainName`" -Force" - Write-Output "`nSetting C$i DNS server to DC1..." - az vm run-command invoke ` + Write-Output "`nAdding C$i to the domain..." + $addToDomainResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name C$i ` + --scripts "`$Password = ConvertTo-SecureString `"$VMPassword`" -AsPlainText -Force; ` + `$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $DomainName\$VMAdmin, `$Password; ` + Add-Computer -DomainName $DomainName -Credential `$Credential -Restart" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addToDomainResponse") + + # The following command fixes this issue: + # https://serverfault.com/questions/754012/windows-10-unable-to-access-sysvol-and-netlogon + Write-Output "`nModifying C$i register to allow access to sysvol..." + $addToSysvolResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name C$i ` + --scripts "cmd.exe /c `"%COMSPEC% /C reg add HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\NetworkProvider\HardenedPaths /v \\*\SYSVOL /d RequireMutualAuthentication=0 /t REG_SZ`"" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addToSysvolResponse") + } +} + +Write-Output $ProcessSeparator +Write-Output "`nVM login info:" +Write-Output "ResourceGroup: $( $ResourceGroup )" +Write-Output "Username: $( $VMAdmin )" +Write-Output "Password: $( $VMPassword )" +Write-Output "SAVE THE ABOVE INFO`n" +Write-Output $ProcessSeparator + +if (-Not $LinuxOnly){ + Write-Output "`nAdding DNS entry for Linux server..." + Write-Warning "NOTE: To verify, log on to DC1 and run 'Resolve-DnsName ls1' in PowerShell. + If it returns NXDOMAIN, you'll need to add it manually." + Write-Output "The time is $( Get-Date )." + # Define the PowerShell script with the DomainName variable interpolated + $scriptContent = @" +`$scriptBlock = { + Add-DnsServerResourceRecordA -Name LS1 -ZoneName $DomainName. -AllowUpdateAny -IPv4Address $LsIP -TimeToLive 01:00:00 -AsJob +} +`$job = Start-Job -ScriptBlock `$scriptBlock +`$timeout = 90 +if (Wait-Job -Job `$job -Timeout `$timeout) { + Receive-Job -Job `$job + Write-Host 'The script completed within the timeout period.' +} else { + Stop-Job -Job `$job + Remove-Job -Job `$job + Write-Host 'The script timed out after `$timeout seconds.' +} +"@ + + # Convert the script to a Base64-encoded string + $bytes = [System.Text.Encoding]::Unicode.GetBytes($scriptContent) + $encodedScript = [Convert]::ToBase64String($bytes) + + + # Run the encoded script on the Azure VM + Write-Output "`nAdding script to add DNS entry for Linux server. No output expected..." + $createDnsScriptResponse = az vm run-command invoke ` --command-id RunPowerShellScript ` + --name DC1 ` --resource-group $ResourceGroup ` - --name C$i ` - --scripts "Get-Netadapter | Set-DnsClientServerAddress -ServerAddresses $DcIP" + --scripts "Set-Content -Path 'C:\AddDnsRecord.ps1' -Value ([System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String('$encodedScript')))" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$createDnsScriptResponse") - Write-Output "`nRestarting C$i..." - az vm restart ` - --resource-group $ResourceGroup ` - --name C$i ` - Write-Output "`nAdding C$i to the domain..." - az vm run-command invoke ` + Write-Output "`nRunning script to add DNS entry for Linux server. It could time out or not. Check output of the next command..." + $addDnsRecordResponse = az vm run-command invoke ` --command-id RunPowerShellScript ` + --name DC1 ` --resource-group $ResourceGroup ` - --name C$i ` - --scripts "`$Password = ConvertTo-SecureString `"$VMPassword`" -AsPlainText -Force; ` - `$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $DomainName\$VMAdmin, `$Password; ` - Add-Computer -DomainName $DomainName -Credential `$Credential -Restart" + --scripts "C:\AddDnsRecord.ps1" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addDnsRecordResponse") - # The following command fixes this issue: - # https://serverfault.com/questions/754012/windows-10-unable-to-access-sysvol-and-netlogon - Write-Output "`nModifying C$i register to allow access to sysvol..." - az vm run-command invoke ` + Write-Host "Checking if ls1 resolves. This should resolve to ls1.lme.local->${LsIP}, not another domain..." + $resolveLs1Response = az vm run-command invoke ` --command-id RunPowerShellScript ` --resource-group $ResourceGroup ` - --name C$i ` - --scripts "cmd.exe /c `"%COMSPEC% /C reg add HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\NetworkProvider\HardenedPaths /v \\*\SYSVOL /d RequireMutualAuthentication=0 /t REG_SZ`"" -} + --name DC1 ` + --scripts "Resolve-DnsName ls1" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$resolveLs1Response") -Write-Output "`nVM login info:" -Write-Output "Username: $($VMAdmin)" -Write-Output "Password: $($VMPassword)" -Write-Output "SAVE THE ABOVE INFO`n" + Write-Host "Removing the Dns script. No output expected..." + $removeDnsRecordScriptResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --name DC1 ` + --resource-group $ResourceGroup ` + --scripts "Remove-Item -Path 'C:\AddDnsRecord.ps1' -Force" + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$removeDnsRecordScriptResponse") -Write-Output "`nAdding DNS entry for Linux server..." -Write-Warning "NOTE: Sometimes this final call hangs indefinitely. -Haven't figured out why. If it doesn't finish after a few minutes, -hit ctrl+c to kill the process. Even if it didn't exit normally, -it is likely that the DNS entry was still successfully added. To -verify, log on to DC1 and run 'Resolve-DnsName ls1' in PowerShell. -If it returns NXDOMAIN, you'll need to add it manually." -Write-Output "The time is $(Get-Date)." -az vm run-command create ` - --resource-group $ResourceGroup ` - --location $Location ` - --run-as-user $DomainName\$VMAdmin ` - --run-as-password $VMPassword ` - --run-command-name "addDNSRecord" ` - --vm-name DC1 ` - --script "Add-DnsServerResourceRecordA -Name `"LS1`" -ZoneName $DomainName -AllowUpdateAny -IPv4Address $LsIP -TimeToLive 01:00:00" +} Write-Output "Done." diff --git a/testing/configure/azure_scripts/copy_file_to_container.ps1 b/testing/configure/azure_scripts/copy_file_to_container.ps1 new file mode 100644 index 00000000..bac7ec33 --- /dev/null +++ b/testing/configure/azure_scripts/copy_file_to_container.ps1 @@ -0,0 +1,81 @@ +<# +.SYNOPSIS +Uploads a file to an Azure Blob Storage container and outputs the SAS URL. + +.DESCRIPTION +This script uploads a specified file to a given Azure Blob Storage container and generates a Shared Access Signature (SAS) URL for the uploaded item. +It requires the local file path, container name, storage account name, and storage account key as mandatory parameters. +This script is useful for automating the process of uploading files to Azure Blob Storage and obtaining a SAS URL for accessing the uploaded file. + +.PARAMETER LocalFilePath +The full local file path of the file to be uploaded. + +.PARAMETER ContainerName +The name of the Azure Blob Storage container where the file will be uploaded. + +.PARAMETER StorageAccountName +The name of the Azure Storage account. + +.PARAMETER StorageAccountKey +The key for the Azure Storage account. + +.OUTPUTS +Shared Access Signature (SAS) URL of the uploaded file. + +.EXAMPLE +.\copy_file_to_container.ps1 -LocalFilePath "C:\path\to\file.txt" -ContainerName "examplecontainer" -StorageAccountName "examplestorageaccount" -StorageAccountKey "examplekey" + +This example uploads 'file.txt' from the local path to 'examplecontainer' in the Azure Storage account named 'examplestorageaccount' and outputs the SAS URL for the uploaded file. + +.NOTES +- Ensure that the Azure CLI is installed and configured with the necessary permissions to access the specified Azure Storage account and container. +- The SAS URL provides access to the file with read permissions and is valid for 1 day. +#> + +param( + [Parameter(Mandatory = $true)] + [string]$LocalFilePath, + + [Parameter(Mandatory = $true)] + [string]$ContainerName, + + [Parameter(Mandatory = $true)] + [string]$StorageAccountName, + + [Parameter(Mandatory = $true)] + [string]$StorageAccountKey +) + +# Upload file to the blob container +$UploadResponse = az storage blob upload ` + --container-name $ContainerName ` + --file $LocalFilePath ` + --name (Split-Path $LocalFilePath -Leaf) ` + --account-name $StorageAccountName ` + --account-key $StorageAccountKey ` + --overwrite + +# Write the upload response to the standard output stream +Write-Host $UploadResponse + +$BlobName = (Split-Path $LocalFilePath -Leaf) +$ExpiryTime = (Get-Date).AddDays(1).ToString('yyyy-MM-ddTHH:mm:ssZ') + +# Generate SAS URL for the blob +$SasUrl = az storage blob generate-sas ` + --account-name $StorageAccountName ` + --account-key $StorageAccountKey ` + --container-name $ContainerName ` + --name $BlobName ` + --permissions r ` + --expiry $ExpiryTime ` + --output tsv + +# Write the SAS URL generation response to the standard output stream +Write-Host "SAS URL generated successfully." + +# Set the full url var for returning to the user for use in the next script +$FullUrl = "https://${StorageAccountName}.blob.core.windows.net/${ContainerName}/${BlobName}?${SasUrl}" + +# Output the FullUrl to the success output stream +Write-Output $FullUrl diff --git a/testing/configure/azure_scripts/create_blob_container.ps1 b/testing/configure/azure_scripts/create_blob_container.ps1 new file mode 100644 index 00000000..8aeb7907 --- /dev/null +++ b/testing/configure/azure_scripts/create_blob_container.ps1 @@ -0,0 +1,95 @@ +<# +.SYNOPSIS +This script creates a new Azure Storage Account and Blob Container within a specified Azure Resource Group. + +.DESCRIPTION +Automates the creation of a unique Azure Storage Account and Blob Container. +Requires the Azure Resource Group name as a mandatory argument. +Generates unique names for the storage account and container, creates the storage account, retrieves the storage account key, +creates a blob container, and saves the configuration to a 'config.ps1' file in the script's directory. + +.PARAMETER ResourceGroup +The name of the Azure Resource Group for the storage account and blob container. + +.EXAMPLE +.\create_blob_container.ps1 -ResourceGroup "YourResourceGroupName" + +Replace "YourResourceGroupName" with the name of your Azure Resource Group. + +.NOTES +- Requires Azure CLI and Azure account login. +- Ensure appropriate permissions in Azure. +- Handle the generated 'config.ps1' file securely. + +#> + + +param( + [Parameter(Mandatory=$true)] + [string]$ResourceGroup +) + +function New-AzureName { + param ( + [Parameter(Mandatory=$true)] + [string]$Prefix + ) + + # Ensuring the prefix is lowercase as Azure Storage Account names must be all lowercase + $Prefix = $Prefix.ToLower() + + # Generate a string of random lowercase letters and numbers + $randomCharacters = -join ((48..57) + (97..122) | Get-Random -Count (24 - $Prefix.Length) | ForEach-Object { [char]$_ }) + + return $Prefix + $randomCharacters +} + +# Get the location of the resource group +$Location = (az group show --name $ResourceGroup --query location --output tsv) + +# Generate a unique storage account name +$StorageAccountName = New-AzureName -Prefix "st" + +# Generate a container name +$ContainerName = New-AzureName -Prefix "container" + +# Create a new storage account +az storage account create ` + --name $StorageAccountName ` + --resource-group $ResourceGroup ` + --location $Location ` + --sku Standard_LRS + +# Wait for a moment to ensure the storage account is available +Start-Sleep -Seconds 10 + +# Get the storage account key +$StorageAccountKey = (az storage account keys list ` + --resource-group $ResourceGroup ` + --account-name $StorageAccountName ` + --query '[0].value' ` + --output tsv) + +# Create a blob container +az storage container create ` + --name $ContainerName ` + --account-name $StorageAccountName ` + --account-key $StorageAccountKey + +# Output the created resources' details +Write-Output "Created Storage Account: $StorageAccountName" +Write-Output "StorageAccountKey: $StorageAccountKey" +Write-Output "Created Container: $ContainerName" + +# Define the file path in the same directory as the running script +$filePath = Join-Path -Path $PSScriptRoot -ChildPath "config.ps1" + +# Write the variables as PowerShell script to the file +@" +`$StorageAccountName = '$StorageAccountName' +`$StorageAccountKey = '$StorageAccountKey' +`$ContainerName = '$ContainerName' +"@ | Set-Content -Path $filePath + + + diff --git a/testing/configure/azure_scripts/download_in_container.ps1 b/testing/configure/azure_scripts/download_in_container.ps1 new file mode 100644 index 00000000..33926357 --- /dev/null +++ b/testing/configure/azure_scripts/download_in_container.ps1 @@ -0,0 +1,106 @@ +<# +.SYNOPSIS +This script automates the file download process on a specified VM based on its OS type. + +.DESCRIPTION +The script takes parameters for VM name, resource group, file URL, destination file path, username, and OS type. It processes these parameters to download a file to a VM, either running Windows or Linux. The script determines the appropriate command to create a directory (if necessary) and download the file to the specified VM, handling differences in command syntax and file path conventions based on the OS. + +.PARAMETER VMName +The name of the Virtual Machine where the file will be downloaded. + +.PARAMETER ResourceGroup +The name of the Azure resource group where the VM is located. + +.PARAMETER FileDownloadUrl +The URL of the file to be downloaded. + +.PARAMETER DestinationFilePath +The complete path where the file should be downloaded on the VM. This path is processed to extract just the filename. + +.PARAMETER UserName +The username for the VM, used in constructing the file path for Linux systems. Default is 'admin.ackbar'. + +.PARAMETER Os +The operating system type of the VM. Accepts 'Windows', 'Linux', or 'linux'. Default is 'Windows'. + +.EXAMPLE +.\download_in_container.ps1 ` + -VMName "MyVM" ` + -ResourceGroup "MyResourceGroup" ` + -FileDownloadUrl "http://example.com/file.zip" ` + -DestinationFilePath "C:\path\to\file.zip" ` + -UserName "admin.ackbar" ` + -Os "Windows" ` + +This example downloads a file from 'http://example.com/file.zip' to 'C:\path\to\file.zip' + on the VM named 'MyVM' in the 'MyResourceGroup'. + +.NOTES +- Ensure that the Azure CLI is installed and configured with the necessary permissions to access and run commands on the specified Azure VM. +- The specified script must exist on the VM and the VM should have the necessary permissions to execute it. +#> + +param( + [Parameter(Mandatory=$true)] + [string]$VMName, + + [Parameter(Mandatory=$true)] + [string]$ResourceGroup, + + [Parameter(Mandatory=$true)] + [string]$FileDownloadUrl, + + [Parameter(Mandatory=$true)] + [string]$DestinationFilePath, # This will be stripped to only the filename + + [Parameter()] + [string]$UserName = "admin.ackbar", + + [Parameter()] + [ValidateSet("Windows","Linux","linux")] + [string]$Os = "Windows" +) + +# Convert the OS parameter to lowercase for consistent comparison +$Os = $Os.ToLower() + +# Extract just the filename from the destination file path +$DestinationFileName = Split-Path -Leaf $DestinationFilePath + +# Set the destination path depending on the OS +if ($Os -eq "linux") { + $DestinationPath = "/home/$UserName/lme/$DestinationFileName" + # Create the lme directory if it doesn't exist + $DirectoryCreationScript = "mkdir -p '/home/$UserName/lme'" + # TODO: We don't want to output this until we fix it so we can put all of the output from thw whole script into one json object + # We are just ignoring the output for now + $CreateDirectoryResponse = az vm run-command invoke ` + --command-id RunShellScript ` + --resource-group $ResourceGroup ` + --name $VMName ` + --scripts $DirectoryCreationScript +} else { + $DestinationPath = "C:\lme\$DestinationFileName" +} + +# The download script +$DownloadScript = if ($Os -eq "linux") { + "curl -o '$DestinationPath' '$FileDownloadUrl'" +} else { + "Invoke-WebRequest -Uri '$FileDownloadUrl' -OutFile '$DestinationPath'" +} + +# Execute the download script with the appropriate command based on OS +if ($Os -eq "linux") { + az vm run-command invoke ` + --command-id RunShellScript ` + --resource-group $ResourceGroup ` + --name $VMName ` + --scripts $DownloadScript +} else { + az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name $VMName ` + --scripts $DownloadScript +} diff --git a/testing/configure/azure_scripts/extract_archive.ps1 b/testing/configure/azure_scripts/extract_archive.ps1 new file mode 100644 index 00000000..4deabcb0 --- /dev/null +++ b/testing/configure/azure_scripts/extract_archive.ps1 @@ -0,0 +1,90 @@ +<# +.SYNOPSIS +Unzips a file on a specified Azure Virtual Machine. + +.DESCRIPTION +This script unzips a specified zip file on an Azure Virtual Machine (VM). It takes the VM's username and a filename (with optional path), +strips the path, constructs the full paths in the VM's 'Downloads' directory, strips the extension from the filename for the extraction path, +and unzips the file. The script requires the VM name, resource group name, username on the VM, and the filename of the zip file. + +.PARAMETER VMName +The name of the Azure Virtual Machine where the file will be unzipped. + +.PARAMETER ResourceGroup +The name of the Azure Resource Group that contains the VM. + +.PARAMETER Filename +The name (and optional path) of the zip file to be unzipped. + +.EXAMPLE +.\extract_archive.ps1 ` + -VMName "DC1" ` + -ResourceGroup "YourResourceGroupName" ` + -Filename "filename.zip" ` + -UserName "admin.ackbar" ` + -Os "Windows" + +This example unzips 'filename.zip' from the 'Downloads' directory of the user 'username' on the VM "DC1" in the resource group "YourResourceGroupName", and extracts it to a subdirectory named 'filename'. + +.NOTES +- Ensure that the Azure CLI is installed and configured with the necessary permissions to access and run commands on the specified Azure VM. +- The VM should have the necessary permissions to read the zip file and write to the extraction directory. +#> + +param( + [Parameter(Mandatory=$true)] + [string]$VMName, + + [Parameter(Mandatory=$true)] + [string]$ResourceGroup, + + [Parameter(Mandatory=$true)] + [string]$Filename, + + [Parameter()] + [string]$UserName = "admin.ackbar", + + [Parameter()] + [ValidateSet("Windows","Linux","linux")] + [string]$Os = "Windows" +) + +# Convert the OS parameter to lowercase for consistent comparison +$Os = $Os.ToLower() + +# Extract just the filename (ignoring any provided path) +$JustFilename = Split-Path -Leaf $Filename + +# Set paths depending on the OS +if ($Os -eq "linux") { + $ZipFilePath = "/home/$UserName/lme/$JustFilename" + $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($JustFilename) + $ExtractToPath = "/home/$UserName/lme/$FileBaseName" # Extract to a subdirectory + + $UnzipScript = @" + unzip '$ZipFilePath' -d '$ExtractToPath' +"@ +} else { + $ZipFilePath = "C:\lme\$JustFilename" + $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($JustFilename) + $ExtractToPath = "C:\lme\$FileBaseName" # Extract to a subdirectory + + $UnzipScript = @" + Expand-Archive -Path '$ZipFilePath' -DestinationPath '$ExtractToPath' +"@ +} + +# Execute the unzip script with the appropriate command based on OS +if ($Os -eq "linux") { + az vm run-command invoke ` + --command-id RunShellScript ` + --resource-group $ResourceGroup ` + --name $VMName ` + --scripts $UnzipScript +} else { + az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name $VMName ` + --scripts $UnzipScript +} diff --git a/testing/configure/azure_scripts/lib/utilityFunctions.ps1 b/testing/configure/azure_scripts/lib/utilityFunctions.ps1 new file mode 100644 index 00000000..838d157c --- /dev/null +++ b/testing/configure/azure_scripts/lib/utilityFunctions.ps1 @@ -0,0 +1,143 @@ +function Format-AzVmRunCommandOutput { + param ( + [Parameter(Mandatory = $true)] + [string]$JsonResponse + ) + + $results = @() + + try { + $responseObj = $JsonResponse | ConvertFrom-Json +# Write-Output "Converted JSON object: $responseObj" + + if ($responseObj -and $responseObj.value) { + $stdout = "" + $stderr = "" + + foreach ($item in $responseObj.value) { +# Write-Output "Processing item: $($item.code)" + + # Check for StdOut and StdErr + if ($item.code -like "ComponentStatus/StdOut/*") { + $stdout += $item.message + "`n" + } elseif ($item.code -like "ComponentStatus/StdErr/*") { + $stderr += $item.message + "`n" + } + + # Additional case to handle other types of 'code' + # This ensures that all messages are captured + else { + $stdout += $item.message + "`n" + } + } + + if ($stdout -or $stderr) { + $results += New-Object PSObject -Property @{ + StdOut = $stdout + StdErr = $stderr + } + } + } + } catch { + $errorMessage = $_.Exception.Message + Write-Output "Error: $errorMessage" + $results += New-Object PSObject -Property @{ + StdOut = "Error: $errorMessage" + StdErr = "" + } + } + + if (-not $results) { + $results += New-Object PSObject -Property @{ + StdOut = "No data or invalid data received." + StdErr = "" + } + } + + return $results +} + +function Show-FormattedOutput { + param ( + [Parameter(Mandatory = $true)] + [Object[]]$FormattedOutput + ) + + foreach ($item in $FormattedOutput) { + if ($item -is [string]) { + # Handle string messages (like error or informational messages) + Write-Output $item + } + elseif ($item -is [PSCustomObject]) { + # Handle custom objects with StdOut and StdErr + if (![string]::IsNullOrWhiteSpace($item.StdOut)) { + Write-Output "Output (stdout):" + Write-Output $item.StdOut + } + if (![string]::IsNullOrWhiteSpace($item.StdErr)) { + Write-Output "Error (stderr):" + Write-Output $item.StdErr + } + } + } +} + +function Get-PrivateKeyFromJson { + param ( + [Parameter(Mandatory = $true)] + [string]$jsonResponse + ) + + try { + # Convert the JSON string to a PowerShell object + $responseObj = $jsonResponse | ConvertFrom-Json + + # Extract the 'message' field + $message = $responseObj.value[0].message + + # Define the start and end markers for the private key + $startMarker = "-----BEGIN OPENSSH PRIVATE KEY-----" + $endMarker = "-----END OPENSSH PRIVATE KEY-----" + + # Find the positions of the start and end markers + $startPosition = $message.IndexOf($startMarker) + $endPosition = $message.IndexOf($endMarker) + + if ($startPosition -lt 0 -or $endPosition -lt 0) { + Write-Error "Private key markers not found in the JSON response." + return $null + } + + # Extract the private key, including the markers + $privateKey = $message.Substring($startPosition, $endPosition - $startPosition + $endMarker.Length) + + # Return the private key + return $privateKey + } + catch { + Write-Error "An error occurred while extracting the private key: $_" + return $null + } +} + +function Invoke-GPUpdateOnVMs { + param( + [Parameter(Mandatory = $true)] + [string]$ResourceGroup, + [int]$numberOfClients = 2 + ) + + for ($i = 1; $i -le $numberOfClients; $i++) { + $vmName = "C$i" # Dynamically create VM name + + # Invoke the command on the VM + $gpupdateResponse = az vm run-command invoke ` + --command-id RunPowerShellScript ` + --name $vmName ` + --resource-group $ResourceGroup ` + --scripts "gpupdate /force" + + # Call the existing Show-FormattedOutput function + Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$gpupdateResponse") + } +} diff --git a/testing/configure/azure_scripts/run_script_in_container.ps1 b/testing/configure/azure_scripts/run_script_in_container.ps1 new file mode 100644 index 00000000..67d15c5f --- /dev/null +++ b/testing/configure/azure_scripts/run_script_in_container.ps1 @@ -0,0 +1,59 @@ +<# +.SYNOPSIS +Executes a specified PowerShell script with arguments on an Azure Virtual Machine. + +.DESCRIPTION +This script remotely executes a PowerShell script that is already present on an Azure Virtual Machine (VM), +passing specified arguments to it. It uses Azure's 'az vm run-command invoke' to run the specified script +located on the VM. The script requires the VM name, resource group name, the full path of the script on the VM, +and a string of arguments to pass to the script. + +.PARAMETER ResourceGroup +The name of the Azure Resource Group that contains the VM. + +.PARAMETER VMName +The name of the Azure Virtual Machine where the script will be executed. + +.PARAMETER ScriptPathOnVM +The full path of the PowerShell script on the Azure VM that needs to be executed. + +.PARAMETER ScriptArguments +A string of arguments that will be passed to the script. + +.EXAMPLE +.\run_script_in_container.ps1 ` + -ResourceGroup "YourResourceGroupName" ` + -VMName "VMName" ` + -ScriptPathOnVM "C:\path\to\your\script.ps1" ` + -ScriptArguments "-Arg1 value1 -Arg2 value2" + +This example executes a script located at 'C:\path\to\your\script.ps1' on the VM named "VMName" + in the resource group "YourResourceGroup", passing it the arguments "-Arg1 value1 -Arg2 value2". + +.NOTES +- Ensure that the Azure CLI is installed and configured with the necessary permissions to access and run commands on the specified Azure VM. +- The specified script must exist on the VM and the VM should have the necessary permissions to execute it. +#> + +param( + [Parameter(Mandatory=$true)] + [string]$ResourceGroup, + + [Parameter(Mandatory=$true)] + [string]$VMName, + + [Parameter(Mandatory=$true)] + [string]$ScriptPathOnVM, # The full path of the script on the VM + + [string]$ScriptArguments # Arguments to pass to the script +) + +$InvokeScriptCommand = @" +& '$ScriptPathOnVM' $ScriptArguments +"@ + +az vm run-command invoke ` + --command-id RunPowerShellScript ` + --resource-group $ResourceGroup ` + --name $VMName ` + --scripts $InvokeScriptCommand diff --git a/testing/configure/azure_scripts/zip_my_parents_parent.ps1 b/testing/configure/azure_scripts/zip_my_parents_parent.ps1 new file mode 100644 index 00000000..ef034496 --- /dev/null +++ b/testing/configure/azure_scripts/zip_my_parents_parent.ps1 @@ -0,0 +1,34 @@ +<# +.SYNOPSIS +Zips the parent of the parent directory of the script and outputs the path of the ZIP file. + +.DESCRIPTION +This script compresses the parent directory of the parent of its location into a ZIP file. +It then outputs the full path of the created ZIP file. This is useful for quickly archiving the contents of the parent directory. + +.EXAMPLE +This example demonstrates how to execute the script and capture the path of the created ZIP file. +# Define the path to this zip script +$zipScriptPath = "C:\path\to\zip_my_parents_parent.ps1" + +# Execute the zip script and capture the output (filename of the zip file) +$zipFilePath = & $zipScriptPath + +.NOTES +- Ensure that PowerShell 5.0 or later is installed, as this script uses the Compress-Archive cmdlet. +- The script assumes read and write permissions in the script's and its parent directory. +#> +# Get the full path of the script's parent directory +$scriptParentDir = Split-Path -Parent $PSScriptRoot + +# Get the name of the parent directory +$parentDirName = Split-Path -Leaf $scriptParentDir + +# Define the destination path for the zip file (adjacent to the parent directory) +$destinationZipPath = Join-Path -Path (Split-Path -Parent $scriptParentDir) -ChildPath ("$parentDirName.zip") + +# Create the zip file +Compress-Archive -Path "$scriptParentDir\*" -DestinationPath $destinationZipPath -Force + +# Output the path of the created zip file +$destinationZipPath diff --git a/testing/configure/chown_dc1_private_key.ps1 b/testing/configure/chown_dc1_private_key.ps1 new file mode 100644 index 00000000..77aa76f3 --- /dev/null +++ b/testing/configure/chown_dc1_private_key.ps1 @@ -0,0 +1,21 @@ +# Path to the private key +$PrivateKeyPath = "C:\lme\id_rsa" + +# Define the SYSTEM account +$SystemAccount = New-Object System.Security.Principal.NTAccount("NT AUTHORITY", "SYSTEM") + +# Get the current ACL of the file +$Acl = Get-Acl -Path $PrivateKeyPath + +# Clear any existing Access Rules +$Acl.SetAccessRuleProtection($true, $false) +$Acl.Access | ForEach-Object { $Acl.RemoveAccessRule($_) | Out-Null } + +# Create a new Access Rule granting FullControl to SYSTEM +$accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule($SystemAccount, "FullControl", "Allow") + +# Add the Access Rule to the ACL +$Acl.AddAccessRule($accessRule) + +# Set the updated ACL back to the file +Set-Acl -Path $PrivateKeyPath -AclObject $Acl diff --git a/testing/configure/create_lme_directory.ps1 b/testing/configure/create_lme_directory.ps1 new file mode 100644 index 00000000..3a4bf5ed --- /dev/null +++ b/testing/configure/create_lme_directory.ps1 @@ -0,0 +1,27 @@ +# Define the directory path +param( + [string]$DirectoryPath = "C:\lme" +) + +# Create the directory if it doesn't already exist +if (-not (Test-Path -Path $DirectoryPath)) { + New-Item -Path $DirectoryPath -ItemType Directory +} + +# Define the security principal for 'All Users' +$allUsers = New-Object System.Security.Principal.SecurityIdentifier("S-1-1-0") + +# Get the current ACL of the directory +$acl = Get-Acl -Path $DirectoryPath + +# Define the rights (read and execute) +$rights = [System.Security.AccessControl.FileSystemRights]::ReadAndExecute + +# Create the rule (allowing read and execute access) +$accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule($allUsers, $rights, 'ContainerInherit, ObjectInherit', 'None', 'Allow') + +# Add the rule to the ACL +$acl.AddAccessRule($accessRule) + +# Set the ACL back to the directory +Set-Acl -Path $DirectoryPath -AclObject $acl diff --git a/testing/configure/create_ou.ps1 b/testing/configure/create_ou.ps1 new file mode 100644 index 00000000..5b546f3c --- /dev/null +++ b/testing/configure/create_ou.ps1 @@ -0,0 +1,23 @@ +param( + [string]$Domain = "lme.local", + [string]$ClientOUCustomName = "LMEClients" +) + +Import-Module ActiveDirectory + +# Split the domain into parts and construct the ParentContainerDN +$DomainParts = $Domain -split "\." +$ParentContainerDN = ($DomainParts | ForEach-Object { "DC=$_" }) -join "," + + +# Define the distinguished name (DN) for the new OU +$NewOUDN = "OU=$ClientOUCustomName,$ParentContainerDN" + +# Check if the OU already exists +if (-not (Get-ADOrganizationalUnit -Filter "DistinguishedName -eq '$NewOUDN'" -ErrorAction SilentlyContinue)) { + # Create the new OU + New-ADOrganizationalUnit -Name $ClientOUCustomName -Path $ParentContainerDN + Write-Output "Organizational Unit '$ClientOUCustomName' created successfully under $ParentContainerDN." +} else { + Write-Output "Organizational Unit '$ClientOUCustomName' already exists under $ParentContainerDN." +} diff --git a/testing/configure/download_files.ps1 b/testing/configure/download_files.ps1 new file mode 100644 index 00000000..1bc6588e --- /dev/null +++ b/testing/configure/download_files.ps1 @@ -0,0 +1,23 @@ +param( + [string]$Directory = $env:USERPROFILE +) + +# Base directory path - use provided username or default to USERPROFILE +$BaseDirectoryPath = if ($Directory -and ($Directory -ne $env:USERPROFILE)) { + "C:\$Directory" +} else { + "$env:USERPROFILE\Downloads\" +} + +# Todo: Allow for downloading a version by adding a parameter for the version number +$ApiUrl = "https://api.github.com/repos/cisagov/LME/releases/latest" +$latestRelease = Invoke-RestMethod -Uri $ApiUrl +$zipFileUrl = $latestRelease.assets | Where-Object { $_.content_type -eq 'application/zip' } | Select-Object -ExpandProperty browser_download_url +$downloadPath = "$BaseDirectoryPath\" + $latestRelease.name + ".zip" +$extractPath = "$BaseDirectoryPath\LME" + +Invoke-WebRequest -Uri $zipFileUrl -OutFile $downloadPath +if (-not (Test-Path -Path $extractPath)) { + New-Item -ItemType Directory -Path $extractPath +} +Expand-Archive -LiteralPath $downloadPath -DestinationPath $extractPath diff --git a/testing/configure/install_chapter_1.ps1 b/testing/configure/install_chapter_1.ps1 new file mode 100644 index 00000000..ee16a0a4 --- /dev/null +++ b/testing/configure/install_chapter_1.ps1 @@ -0,0 +1,65 @@ +param ( + [Parameter( + HelpMessage="Path to the configuration directory. Default is 'C:\lme\configure'." + )] + [string]$ConfigurePath = "C:\lme\configure", + [Parameter( + HelpMessage="Path to the root install directory. Default is 'C:\lme'." + )] + [string]$RootInstallDir = "C:\lme" + +) + +# Exit the script on any error +$ErrorActionPreference = 'Stop' +$ProcessSeparator = "`n----------------------------------------`n" + +# Change directory to the configure directory +Set-Location -Path $ConfigurePath + +# Run the scripts and check for failure +Write-Output "Creating the configurePath directory..." +.\create_lme_directory.ps1 -DirectoryPath $RootInstallDir +Write-Output $ProcessSeparator + +Write-Output "Downloading the files..." +.\download_files.ps1 -Directory lme +Write-Output $ProcessSeparator + +Write-Output "Importing the GPOs..." +.\wec_import_gpo.ps1 -Directory lme +Write-Output $ProcessSeparator + +Start-Sleep 10 +Write-Output "Updating the GPO server name..." +.\wec_gpo_update_server_name.ps1 +Write-Output $ProcessSeparator + +Write-Output "Creating the OU..." +.\create_ou.ps1 +Write-Output $ProcessSeparator + +Write-Output "Linking the GPOs..." +.\wec_link_gpo.ps1 +Write-Output $ProcessSeparator + +Write-Output "Provisioning the WEC service..." +.\wec_service_provisioner.ps1 +Write-Output $ProcessSeparator + +# Run the wevtutil and wecutil commands +Write-Output "Running wevtutil and wecutil commands to start the wec service manually..." +wevtutil set-log ForwardedEvents /q:true /e:true +Write-Output $ProcessSeparator + +Write-Output "Running wecutil restart command..." +wecutil rs lme +Write-Output $ProcessSeparator + +Write-Output "Running wecutil gr command..." +wecutil gr lme +Write-Output $ProcessSeparator + +# Run the move_computers_to_ou script +Write-Output "Moving the computers to the OU..." +.\move_computers_to_ou.ps1 diff --git a/testing/configure/install_chapter_2.ps1 b/testing/configure/install_chapter_2.ps1 new file mode 100644 index 00000000..d85a9d6b --- /dev/null +++ b/testing/configure/install_chapter_2.ps1 @@ -0,0 +1,28 @@ +param ( + [Parameter( + HelpMessage="Path to the configuration directory. Default is 'C:\lme\configure'." + )] + [string]$ConfigurePath = "C:\lme\configure" +) + +# Exit the script on any error +$ErrorActionPreference = 'Stop' +$ProcessSeparator = "`n----------------------------------------`n" + +# Change directory to the configure directory +Set-Location -Path $ConfigurePath + +Write-Output "Installing Sysmon..." +.\sysmon_install_in_sysvol.ps1 +Write-Output $ProcessSeparator + +Write-Output "Importing the gpo..." +.\sysmon_import_gpo.ps1 -Directory lme +Write-Output $ProcessSeparator + +Write-Output "Updating the gpo variables.." +.\sysmon_gpo_update_vars.ps1 +Write-Output $ProcessSeparator + +Write-Output "Linking the gpo..." +.\sysmon_link_gpo.ps1 diff --git a/testing/configure/lib/functions.sh b/testing/configure/lib/functions.sh new file mode 100644 index 00000000..c9c013cf --- /dev/null +++ b/testing/configure/lib/functions.sh @@ -0,0 +1,31 @@ +extract_credentials() { + # Set default file path if not provided + local file_path=${1:-'/opt/lme/Chapter 3 Files/output.log'} + + # Check if the file exists + if [ ! -f "$file_path" ]; then + echo "File not found: $file_path" + return 1 + fi + + # Read and extract credentials from the last 18 lines + while IFS= read -r line; do + # Remove leading '## ' and trim whitespaces from the line + line=$(echo "$line" | sed 's/^## //g' | xargs) + + # Split the line into key and value + key=$(echo "$line" | awk -F ':' '{print $1}') + value=$(echo "$line" | awk -F ':' '{print $2}' | xargs) # xargs to trim whitespaces from value + + # Remove non-word characters (keep only word characters) + value=$(echo "$value" | sed 's/[^[:alnum:]_]//g') + + case $key in + "elastic") export elastic=$value ;; + "kibana") export kibana=$value ;; + "logstash_system") export logstash_system=$value ;; + "logstash_writer") export logstash_writer=$value ;; + "dashboard_update") export dashboard_update=$value ;; + esac + done < <(tail -n 18 "$file_path" | grep -E "(elastic|kibana|logstash_system|logstash_writer|dashboard_update):") +} diff --git a/testing/configure/linux_authorize_private_key.sh b/testing/configure/linux_authorize_private_key.sh new file mode 100755 index 00000000..c699d816 --- /dev/null +++ b/testing/configure/linux_authorize_private_key.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +cat /home/admin.ackbar/.ssh/id_rsa.pub >> /home/admin.ackbar/.ssh/authorized_keys +sudo chown admin.ackbar:admin.ackbar /home/admin.ackbar/.ssh/* +perl -p -i -e 's/root\@LS1/admin.ackbar\@DC1/' /home/admin.ackbar/.ssh/authorized_keys diff --git a/testing/configure/linux_install_lme.exp b/testing/configure/linux_install_lme.exp new file mode 100755 index 00000000..a3f9b98f --- /dev/null +++ b/testing/configure/linux_install_lme.exp @@ -0,0 +1,56 @@ +#!/usr/bin/expect + +# Change to the LME directory containing files for the Linux server +cd /opt/lme/Chapter\ 3\ Files/ + +# Adjust the timeout if necessary +set timeout 60 +set expect_out(buffer_size) 1000000 + +log_file -a output.log + +spawn ./deploy.sh install +sleep 1 +expect { + -re {.*OK.*} { + send "\r" + } + -re {.*Proceed.*} { + send "y\r" + } +} + + +expect { + -re {.*Please reboot and re-run this script to finish the install.*} { + send_user "Reboot required. Exiting...\n" + exit + } + -re "Enter the IP of this Linux server.*" { + sleep 1 + send "\r" + } +} + +sleep 1 +expect -re {Windows Event Collector} +sleep 1 +send "ls1.lme.local\r" + +sleep 1 + +expect -re {continue with self signed certificates.*: y} +sleep 1 +send "\r" +sleep 1 + +expect -re {Skip Docker Install\? \(\[y\]es/\[n\]o\): n} +sleep 1 +send "\r" +set timeout 2600 + +expect eof + +log_file + +exec cat output.log diff --git a/testing/configure/linux_install_lme.sh b/testing/configure/linux_install_lme.sh new file mode 100755 index 00000000..7cd71703 --- /dev/null +++ b/testing/configure/linux_install_lme.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Change to the directory where the script is located +script_dir=$(dirname "$0") +cd $script_dir || exit 1 +# We need to get the full path of the script dir for below +script_dir=$(pwd) + +# Default username +username="admin.ackbar" + +# Process command line arguments +while getopts "u:v:b:" opt; do + case $opt in + u) username=$OPTARG ;; + v) version=$OPTARG ;; + b) branch=$OPTARG ;; + \?) echo "Invalid option -$OPTARG" >&2; exit 1 ;; + esac +done + +# Check if version matches the pattern +if [[ -n "$version" && ! $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid version format. Version should match \d+.\d+.\d+" + exit 1 +fi + +# Remove any existing LME directories +sudo rm -rf /opt/cisagov-LME-* /opt/lme + +# Get the tarball URL for the specified version +get_tarball_url() { + echo "https://api.github.com/repos/cisagov/LME/tarball/v$1" +} + +if [ -n "$branch" ]; then + # Clone from the specified branch + git clone --branch "$branch" https://github.com/cisagov/LME.git /opt/lme +else + echo "Getting the code from GitHub" + # Check if a version is provided + if [ -n "$version" ]; then + tarball_url=$(get_tarball_url "$version") + else + tarball_url=$(curl -s https://api.github.com/repos/cisagov/LME/releases/latest | jq -r '.tarball_url') + fi + + # Get the version from the tarball URL + v_version=$(basename "$tarball_url") + + echo "Downloading $tarball_url to file: $v_version" + curl -L "$tarball_url" -o "$v_version" + + # extracts it to a folder like cisagov-LME-3412897 + sudo tar -xzpf "$v_version" -C /opt + rm -rf "$v_version" + + extracted_filename=$(sudo ls -ltd /opt/cisagov-LME-* | grep "^d" | head -n 1 | awk '{print $NF}') + + echo "Extracted to $extracted_filename" + + echo "Renaming directory to /opt/lme" + sudo mv "$extracted_filename" /opt/lme +fi + +echo 'export DEBIAN_FRONTEND=noninteractive' >> ~/.bashrc +echo 'export NEEDRESTART_MODE=a' >> ~/.bashrc +. ~/.bashrc + +# Set the noninteractive modes for root +echo 'export DEBIAN_FRONTEND=noninteractive' | sudo tee -a /root/.bashrc +echo 'export NEEDRESTART_MODE=a' | sudo tee -a /root/.bashrc + +# Execute script with root privileges +# Todo: We could put a switch here for different versions and just run different expect scripts +sudo -E bash -c ". /root/.bashrc && $script_dir/linux_install_lme.exp" + +sudo chmod ugo+w "/opt/lme/Chapter 3 Files/output.log" + +if [ -f "/opt/lme/files_for_windows.zip" ]; then + sudo cp /opt/lme/files_for_windows.zip /home/"$username"/ + sudo chown "$username":"$username" /home/"$username"/files_for_windows.zip +else + echo "files_for_windows.zip does not exist. Probably because a reboot is required in order to proceed with the install" +fi diff --git a/testing/configure/linux_make_private_key.exp b/testing/configure/linux_make_private_key.exp new file mode 100755 index 00000000..16fd6ec9 --- /dev/null +++ b/testing/configure/linux_make_private_key.exp @@ -0,0 +1,16 @@ +#!/usr/bin/expect + +spawn ssh-keygen -t rsa -b 4096 +sleep 1 +expect -re {Enter file in which to save the key} +send "/home/admin.ackbar/.ssh/id_rsa\r" +sleep 1 +expect -re {empty for no passphrase} +send "\r" +sleep 1 +expect -re {Enter same passphrase again} +send "\r" + +set timeout 60 + +expect eof diff --git a/testing/configure/linux_test_install.sh b/testing/configure/linux_test_install.sh new file mode 100755 index 00000000..3dda731d --- /dev/null +++ b/testing/configure/linux_test_install.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -e + +# Get the full path to the directory containing the current script +script_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") + +source "${script_dir}/lib/functions.sh" +extract_credentials '/opt/lme/Chapter 3 Files/output.log' + +check_variable() { + local var_name="$1" + local var_value="$2" + + if [ -z "$var_value" ]; then + echo "Error: '$var_name' is not set or is empty" + return 1 # Return a non-zero status to indicate failure + fi +} + +# Perform the checks +check_variable "elastic" "$elastic" || exit 1 +check_variable "kibana" "$kibana" || exit 1 +check_variable "logstash_system" "$logstash_system" || exit 1 +check_variable "logstash_writer" "$logstash_writer" || exit 1 +check_variable "dashboard_update" "$dashboard_update" || exit 1 + +echo "All variables are set correctly." + +# Get the list of containers and their health status +container_statuses=$(docker ps --format "{{.Names}}: {{.Status}}" | grep -v "CONTAINER ID") + +# Check each container's status +unhealthy=false +while read -r line; do + container_name=$(echo "$line" | awk -F': ' '{print $1}') + health_status=$(echo "$line" | awk -F': ' '{print $2}') + + if [[ $health_status != *"(healthy)"* ]]; then + echo "Container $container_name is not healthy: $health_status" + unhealthy=true + exit 1 + fi +done <<< "$container_statuses" + +# Final check +if [ "$unhealthy" = false ]; then + echo "All containers are healthy." +fi + +ELASTICSEARCH_HOST="localhost" +ELASTICSEARCH_PORT="9200" + +# Get list of all indexes +indexes=$(curl -sk -u "elastic:$elastic" "https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_cat/indices?v" | awk '{print $3}') + +# Check if winlogbeat index exists +if echo "$indexes" | grep -q "winlogbeat"; then + echo "Index 'winlogbeat' exists." +else + echo "Index 'winlogbeat' does not exist." >&2 + exit 1 +fi + +# Check if we can query the winlogbeat index +response=$(curl -sk -u "elastic:$elastic" "https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/winlogbeat-*/_search" -H "Content-Type: application/json" -d '{ + "size": 1, + "query": { + "match_all": {} + } +}') + +# Check if the curl command was successful +if [ $? -eq 0 ]; then + echo "Querying winlogbeat executed successfully." +else + echo "Error executing the query of winlogbeat." >&2 + exit 1 +fi + +# Check the kibana saved objects. +# response=$(curl -sk -u "elastic:$elastic" "https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/.kibana/_search" -H "Content-Type: application/json" -d '{ +# "size": 1000, +# "query": { +# "term": { +# "type": "dashboard" +# } +# } +# }') +# echo $response + + +response=$(curl -sk -u "elastic:$elastic" "https://${ELASTICSEARCH_HOST}/api/kibana/management/saved_objects/_find?perPage=500&page=1&type=dashboard&sortField=updated_at&sortOrder=desc") + +#!/bin/bash + +# List of dashboard names to check +declare -a names_to_check=( + "User Security" + "User HR" + "Sysmon Summary" + "Security Dashboard - Security Log" + "Process Explorer" + "Computer Software Overview" + "Alerting Dashboard" + "HealthCheck Dashboard - Overview" +) + +# Extract dashboard names from the JSON response stored in the variable +dashboard_names=$(echo "$response" | jq -r '.saved_objects[] | select(.type == "dashboard") | .meta.title') + +# Check each name +for name in "${names_to_check[@]}"; do + if grep -qF "$name" <<< "$dashboard_names"; then + echo "Dashboard found: $name" + else + echo "Dashboard NOT found: $name" >&2 + exit 1 + fi +done diff --git a/testing/configure/linux_update_system.sh b/testing/configure/linux_update_system.sh new file mode 100755 index 00000000..602e185d --- /dev/null +++ b/testing/configure/linux_update_system.sh @@ -0,0 +1,3 @@ +# Install Git client to be able to clone the LME repository +sudo apt update +sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt install git curl zip net-tools jq nodejs expect python3-venv -y \ No newline at end of file diff --git a/testing/configure/list_computers_forwarding_events.ps1 b/testing/configure/list_computers_forwarding_events.ps1 new file mode 100644 index 00000000..ecf8ff3a --- /dev/null +++ b/testing/configure/list_computers_forwarding_events.ps1 @@ -0,0 +1,27 @@ +# Execute 'wecutil gr lme' command and capture the output +$wecutilOutput = wecutil gr lme + +# Split the output into individual lines +$lines = $wecutilOutput -split "`r`n" | Where-Object { $_ -match "\S" } # Exclude empty lines + +# Initialize a list to store active computer names +$activeComputers = @() + +# Process each line to extract computer names with active status +$isActive = $false +foreach ($line in $lines) { + if ($line -match "RunTimeStatus: Active") { + $isActive = $true + } elseif ($line -match "\.local") { + if ($isActive) { + if ($line -match "(\S+\.local)") { + $activeComputers += $matches[1] + } + } + $isActive = $false + } +} + +# Display the active computer names +Write-Output "Active Computers Forwarding Events:" +$activeComputers | ForEach-Object { Write-Output $_ } diff --git a/testing/configure/move_computers_to_ou.ps1 b/testing/configure/move_computers_to_ou.ps1 new file mode 100644 index 00000000..4ef36c49 --- /dev/null +++ b/testing/configure/move_computers_to_ou.ps1 @@ -0,0 +1,38 @@ +param( + [string]$Domain = "lme.local", + [string]$ClientOUCustomName = "LMEClients", + [string]$CurrentCN = "Computers" +) + +# Import the Active Directory module +Import-Module ActiveDirectory + +# Split the domain into its parts +$domainParts = $Domain -split '\.' + +# Construct the domain DN, starting with 'DC=' +$domainDN = 'DC=' + ($domainParts -join ',DC=') + +# Define the DN of the existing Computers container +$computersContainerDN = "CN=$CurrentCN,$domainDN" + +# Define the DN of the target OU +$targetOUDN = "OU=$ClientOUCustomName,$domainDN" + +# Output the DNs for verification +Write-Output "Current Computers Container DN: $computersContainerDN" +Write-Output "Target OU DN: $targetOUDN" + +# Get the computer accounts in the Computers container +$computers = Get-ADComputer -Filter * -SearchBase $computersContainerDN + +# Move each computer to the target OU +foreach ($computer in $computers) { + try { + # Move the computer to the target OU + Move-ADObject -Identity $computer.DistinguishedName -TargetPath $targetOUDN + Write-Output "Moved $($computer.Name) to $targetOUDN" + } catch { + Write-Output "Failed to move $($computer.Name): $_" + } +} \ No newline at end of file diff --git a/testing/configure/sysmon_gpo_update_vars.ps1 b/testing/configure/sysmon_gpo_update_vars.ps1 new file mode 100644 index 00000000..9707039a --- /dev/null +++ b/testing/configure/sysmon_gpo_update_vars.ps1 @@ -0,0 +1,43 @@ +param( + [string]$GpoName = "LME-Sysmon-Task", + [string]$DomainName = "lme.local" +) + +# Get the FQDN of the current server +$fqdn = [System.Net.Dns]::GetHostByName($env:COMPUTERNAME).HostName + +# Get the GPO object +$gpo = Get-GPO -Name $GpoName + +# Check if GPO is found +if ($null -eq $gpo) { + Write-Output "GPO not found" + exit +} + +# Get the GUID of the GPO +$gpoGuid = $gpo.Id + +# Define the path to the XML file +$xmlFilePath = "C:\Windows\SYSVOL\sysvol\$DomainName\Policies\{$gpoGuid}\Machine\Preferences\ScheduledTasks\ScheduledTasks.xml" + +# Get current time and add 5 minutes +$newStartTime = (Get-Date).AddMinutes(5).ToString("yyyy-MM-ddTHH:mm:ss") + +# Load the XML file +$xml = [xml](Get-Content -Path $xmlFilePath) + +# Find the task with name "LME-Sysmon-Task" +$task = $xml.ScheduledTasks.TaskV2 | Where-Object { $_.Properties.name -eq "LME-Sysmon-Task" } + +# Update the start time in the XML +$task.Properties.Task.Triggers.CalendarTrigger.StartBoundary = $newStartTime + +# Update the command path +$task.Properties.Task.Actions.Exec.Command = "\\$fqdn\sysvol\$DomainName\LME\Sysmon\update.bat" + +# Save the modified XML back to the file +$xml.Save($xmlFilePath) + +# Output the new start time for verification +Write-Output "New start time set to: $newStartTime" \ No newline at end of file diff --git a/testing/configure/sysmon_import_gpo.ps1 b/testing/configure/sysmon_import_gpo.ps1 new file mode 100644 index 00000000..2f3eb109 --- /dev/null +++ b/testing/configure/sysmon_import_gpo.ps1 @@ -0,0 +1,34 @@ +param( + [string]$Directory = $env:USERPROFILE +) + +# Determine the base directory path based on the provided username +$baseDirectoryPath = if ($Directory -and ($Directory -ne $env:USERPROFILE)) { + "C:\$Directory" +} else { + "$env:USERPROFILE\Downloads" +} + +$GPOBackupPath = "$baseDirectoryPath\LME\Chapter 2 Files\GPO Deployment\Group Policy Objects" + +$gpoNames = @("LME-Sysmon-Task") + +foreach ($gpoName in $gpoNames) { + $gpo = Get-GPO -Name $gpoName -ErrorAction SilentlyContinue + if (-not $gpo) { + New-GPO -Name $gpoName | Out-Null + Write-Output "Created GPO: $gpoName" + } else { + Write-Output "GPO $gpoName already exists." + } + + try { + Import-GPO -BackupGpoName $gpoName -TargetName $gpoName -Path $GPOBackupPath -CreateIfNeeded -ErrorAction Stop + Write-Output "Imported settings into GPO: $gpoName" + } catch { + Throw "Failed to import GPO: $gpoName. The GPODisplayName in bkupinfo.xml may not match or other import error occurred." + } +} + +Write-Output "LME Sysmon GPOs have been created and imported successfully." + diff --git a/testing/configure/sysmon_install_in_sysvol.ps1 b/testing/configure/sysmon_install_in_sysvol.ps1 new file mode 100644 index 00000000..41b59777 --- /dev/null +++ b/testing/configure/sysmon_install_in_sysvol.ps1 @@ -0,0 +1,69 @@ +param( + [string]$DomainName = "lme.local" # Default domain name +) + +# Define the SYSVOL path +$destinationPath = "C:\Windows\SYSVOL\SYSVOL\$DomainName\LME\Sysmon" +$tempPath = Join-Path $env:TEMP "SysmonTemp" + +# Create the LME and Sysmon directories +New-Item -ItemType Directory -Path $destinationPath -Force +New-Item -ItemType Directory -Path $tempPath -Force + +# Copy update.bat from the user's download directory +$updateBatSource = "C:\lme\LME\Chapter 2 Files\GPO Deployment\update.bat" +Copy-Item -Path $updateBatSource -Destination $destinationPath + +# Download URL for Sysmon +$url = "https://download.sysinternals.com/files/Sysmon.zip" + +# Download file path +$zipFilePath = Join-Path $tempPath "Sysmon.zip" + +# Download the file +Invoke-WebRequest -Uri $url -OutFile $zipFilePath + +# Unzip the file to temp directory +Expand-Archive -Path $zipFilePath -DestinationPath $tempPath + +# Copy only Sysmon64.exe to destination +Copy-Item -Path "$tempPath\Sysmon64.exe" -Destination $destinationPath + +# Clean up: remove temp directory and zip file +Remove-Item -Path $tempPath -Recurse -Force + +# Download URL for the Sysmon configuration file +$xmlUrl = "https://raw.githubusercontent.com/SwiftOnSecurity/sysmon-config/master/sysmonconfig-export.xml" + +# Destination file path for the Sysmon configuration file +$xmlFilePath = Join-Path $destinationPath "sysmon.xml" + +# Download and rename the file +Invoke-WebRequest -Uri $xmlUrl -OutFile $xmlFilePath + +# Define the destination path for Sigcheck +$sigcheckDestinationPath = "C:\Windows\SYSVOL\SYSVOL\$DomainName\LME" + +# Download URL for Sigcheck +$sigcheckUrl = "https://download.sysinternals.com/files/Sigcheck.zip" + +# Temporary path for Sigcheck zip file +$sigcheckTempPath = Join-Path $env:TEMP "SigcheckTemp" + +# Ensure the temporary directory exists +New-Item -ItemType Directory -Path $sigcheckTempPath -Force + +# Download file path for Sigcheck +$sigcheckZipFilePath = Join-Path $sigcheckTempPath "Sigcheck.zip" + +# Download the Sigcheck zip file +Invoke-WebRequest -Uri $sigcheckUrl -OutFile $sigcheckZipFilePath + +# Unzip the Sigcheck file to temporary directory +Expand-Archive -Path $sigcheckZipFilePath -DestinationPath $sigcheckTempPath + +# Copy only Sigcheck64.exe to the destination +Copy-Item -Path "$sigcheckTempPath\sigcheck64.exe" -Destination $sigcheckDestinationPath + +# Clean up: remove temporary directory and zip file +Remove-Item -Path $sigcheckTempPath -Recurse -Force diff --git a/testing/configure/sysmon_link_gpo.ps1 b/testing/configure/sysmon_link_gpo.ps1 new file mode 100644 index 00000000..a29aa9eb --- /dev/null +++ b/testing/configure/sysmon_link_gpo.ps1 @@ -0,0 +1,18 @@ +param( + [string]$Domain = "lme.local", + [string]$ClientOUCustomName = "LMEClients" +) + +Import-Module ActiveDirectory + +$domainDN = $Domain -replace '\.', ',DC=' -replace '^', 'DC=' +$OUDistinguishedName = "OU=$ClientOUCustomName,$domainDN" + +$GPOName = "LME-Sysmon-Task" + +try { + New-GPLink -Name $GPOName -Target $OUDistinguishedName + Write-Output "GPO '$GPOName' linked to OU '$ClientOUCustomName'." +} catch { + Write-Output "Error linking GPO '$GPOName' to OU '$ClientOUCustomName': $_" +} diff --git a/testing/configure/trust_ls1_ssh_key.ps1 b/testing/configure/trust_ls1_ssh_key.ps1 new file mode 100644 index 00000000..0f3d0a41 --- /dev/null +++ b/testing/configure/trust_ls1_ssh_key.ps1 @@ -0,0 +1,66 @@ +param ( + [string]$SshHost = "ls1" +) + +$SshDirectory = "C:\Windows\System32\config\systemprofile\.ssh" +$KnownHostsFile = Join-Path -Path $SshDirectory -ChildPath "known_hosts" + +# Ensure the .ssh directory exists +if (-not (Test-Path -Path $SshDirectory)) { + New-Item -ItemType Directory -Path $SshDirectory +} + +# Function to set ACL for the directory, granting FullControl to SYSTEM and applying inheritance +function Set-SystemOnlyAclForDirectory { + param ( + [string]$path + ) + + $systemAccount = New-Object System.Security.Principal.NTAccount("NT AUTHORITY", "SYSTEM") + $acl = Get-Acl -Path $path + $acl.SetAccessRuleProtection($true, $false) # Enable ACL protection, disable inheritance + $acl.Access | ForEach-Object { $acl.RemoveAccessRule($_) | Out-Null } # Clear existing rules + + # Create and add the Access Rule for SYSTEM with inheritance + $accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule($systemAccount, "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow") + $acl.AddAccessRule($accessRule) + + # Apply the updated ACL to the directory + Set-Acl -Path $path -AclObject $acl +} + +# Function to set ACL for a file, granting FullControl only to SYSTEM +function Set-SystemOnlyAclForFile { + param ( + [string]$path + ) + + $systemAccount = New-Object System.Security.Principal.NTAccount("NT AUTHORITY", "SYSTEM") + $acl = Get-Acl -Path $path + $acl.SetAccessRuleProtection($true, $false) # Enable ACL protection, disable inheritance + $acl.Access | ForEach-Object { $acl.RemoveAccessRule($_) | Out-Null } # Clear existing rules + + # Create and add the Access Rule for SYSTEM + $accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule($systemAccount, "FullControl", "Allow") + $acl.AddAccessRule($accessRule) + + # Apply the updated ACL to the file + Set-Acl -Path $path -AclObject $acl +} + +# Set ACL for the .ssh directory with inheritance +Set-SystemOnlyAclForDirectory -path $SshDirectory + +# Ensure the known_hosts file exists +if (-not (Test-Path -Path $KnownHostsFile)) { + New-Item -ItemType File -Path $KnownHostsFile +} + +# Set ACL for the known_hosts file without inheritance +Set-SystemOnlyAclForFile -path $KnownHostsFile + +# Run ssh-keyscan and append output to known_hosts +ssh-keyscan $SshHost | Out-File -FilePath $KnownHostsFile -Append -Encoding UTF8 + +# Output the contents of the known_hosts file +Get-Content -Path $KnownHostsFile diff --git a/testing/configure/wec_firewall.ps1 b/testing/configure/wec_firewall.ps1 new file mode 100644 index 00000000..3e3bb129 --- /dev/null +++ b/testing/configure/wec_firewall.ps1 @@ -0,0 +1,18 @@ +# Asks user to provide subnet - then creates a inbound allow firewall rule for 5985. Run on WEC server. +param ( + [string]$InboundRuleName = "WinRM TCP In 5985", + [string]$ClientSubnet = "10.1.0.0/24", + [string]$LocalPort = "5985" +) + +if (-not (Get-NetFirewallRule -Name $InboundRuleName -ErrorAction SilentlyContinue)) { + New-NetFirewallRule -DisplayName $InboundRuleName ` + -Direction Inbound -Protocol TCP ` + -LocalPort $LocalPort -Action Allow ` + -RemoteAddress $ClientSubnet ` + -Description "Allow inbound TCP ${LocalPort} for WinRM from clients subnet" +} else { + Write-Output "Inbound rule '$InboundRuleName' already exists." +} + +Write-Output "Inbound WinRM rule has been configured." diff --git a/testing/configure/wec_gpo_update_server_name.ps1 b/testing/configure/wec_gpo_update_server_name.ps1 new file mode 100644 index 00000000..6557042b --- /dev/null +++ b/testing/configure/wec_gpo_update_server_name.ps1 @@ -0,0 +1,42 @@ +<# +.SYNOPSIS +This script sets and retrieves a Group Policy (GP) registry value for Windows Event Log Event Forwarding. + +.DESCRIPTION +The script is used to configure the Subscription Manager URL for Windows Event Log Event Forwarding in a Group Policy setting. It sets the registry value for the Subscription Manager URL using the specified domain, port, and protocol, and then retrieves the value to confirm the setting. This is useful in environments where centralized event log management is required. + +.PARAMETER domain +The domain for the Subscription Manager URL. Default is 'dc1.lme.local'. + +.PARAMETER port +The port number for the Subscription Manager URL. Default is 5985. + +.PARAMETER protocol +The protocol for the Subscription Manager URL. Default is 'http'. + +.EXAMPLE +.\wec_gpo_update_server_name.ps1 +Executes the script with default parameters. + +.EXAMPLE +.\wec_gpo_update_server_name.ps1 -Domain "customdomain.local" -Port 1234 -Protocol "https" +Executes the script with custom domain, port, and protocol. + +#> + +param( + [string]$Domain = "dc1.lme.local", + [int]$Port = 5985, + [string]$Protocol = "http" +) + +# Construct the Subscription Manager URL using the provided parameters +$subscriptionManagerUrl = "Server=${Protocol}://${Domain}:${Port}/wsman/SubscriptionManager/WEC,Refresh=60" +Set-GPRegistryValue -Name "LME-WEC-Client" -Key "HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\EventLog\EventForwarding\SubscriptionManager" -Value $subscriptionManagerUrl -Type String + +# To get the GP registry value to confirm it's set +$registryValue = Get-GPRegistryValue -Name "LME-WEC-Client" -Key "HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\EventLog\EventForwarding\SubscriptionManager" + +# Output the retrieved registry value +Write-Output "Set the subscription manager url value to: " +$registryValue diff --git a/testing/configure/wec_import_gpo.ps1 b/testing/configure/wec_import_gpo.ps1 new file mode 100644 index 00000000..8906ce2d --- /dev/null +++ b/testing/configure/wec_import_gpo.ps1 @@ -0,0 +1,34 @@ +param( + [string]$Directory = $env:USERPROFILE +) + +# Determine the base directory path based on the provided username +$BaseDirectoryPath = if ($Directory -and ($Directory -ne $env:USERPROFILE)) { + "C:\$Directory" +} else { + "$env:USERPROFILE\Downloads" +} + +$GPOBackupPath = "$BaseDirectoryPath\LME\Chapter 1 Files\Group Policy Objects" + +$GpoNames = @("LME-WEC-Client", "LME-WEC-Server") + +foreach ($gpoName in $GpoNames) { + $gpo = Get-GPO -Name $gpoName -ErrorAction SilentlyContinue + if (-not $gpo) { + New-GPO -Name $gpoName | Out-Null + Write-Output "Created GPO: $gpoName" + } else { + Write-Output "GPO $gpoName already exists." + } + + try { + Import-GPO -BackupGpoName $gpoName -TargetName $gpoName -Path $GPOBackupPath -CreateIfNeeded -ErrorAction Stop + Write-Output "Imported settings into GPO: $gpoName" + } catch { + Throw "Failed to import GPO: $gpoName. The GPODisplayName in bkupinfo.xml may not match or other import error occurred." + } +} + +Write-Output "LME GPOs have been created and imported successfully." + diff --git a/testing/configure/wec_link_gpo.ps1 b/testing/configure/wec_link_gpo.ps1 new file mode 100644 index 00000000..3c8c4921 --- /dev/null +++ b/testing/configure/wec_link_gpo.ps1 @@ -0,0 +1,27 @@ +param( + [string]$Domain = "lme.local", + [string]$ClientOUCustomName = "LMEClients" +) + +Import-Module ActiveDirectory + +$DomainDN = $Domain -replace '\.', ',DC=' -replace '^', 'DC=' +$ClientOUDistinguishedName = "OU=$ClientOUCustomName,$DomainDN" + +$GPONameClient = "LME-WEC-Client" +$GPONameServer = "LME-WEC-Server" +$ServerOUDistinguishedName = "OU=Domain Controllers,$DomainDN" + +try { + New-GPLink -Name $GPONameClient -Target $ClientOUDistinguishedName + Write-Output "GPO '$GPONameClient' linked to OU '$ClientOUCustomName'." +} catch { + Write-Output "Error linking GPO '$GPONameClient' to OU '$ClientOUCustomName': $_" +} + +try { + New-GPLink -Name $GPONameServer -Target $ServerOUDistinguishedName + Write-Output "GPO '$GPONameServer' linked to OU 'Domain Controllers'." +} catch { + Write-Output "Error linking GPO '$GPONameServer' to OU 'Domain Controllers': $_" +} diff --git a/testing/configure/wec_service_provisioner.ps1 b/testing/configure/wec_service_provisioner.ps1 new file mode 100644 index 00000000..4c0d1302 --- /dev/null +++ b/testing/configure/wec_service_provisioner.ps1 @@ -0,0 +1,24 @@ +# PowerShell script to configure Windows Event Collector + +param( + [string]$XmlFilePath = "C:\lme\LME\Chapter 1 Files\lme_wec_config.xml" +) + +# Check if Windows Event Collector Service is running and start it if not +$wecService = Get-Service -Name "Wecsvc" +if ($wecService.Status -ne 'Running') { + Start-Service -Name "Wecsvc" + Write-Output "Windows Event Collector Service started." +} else { + Write-Output "Windows Event Collector Service is already running." +} + +# Check if the XML configuration file exists +if (Test-Path -Path $XmlFilePath) { + # Run the wecutil command to configure the collector + wecutil cs $XmlFilePath + Write-Output "wecutil command executed successfully with config file: $XmlFilePath" +} else { + Write-Output "Configuration file not found at $XmlFilePath" +} + diff --git a/testing/configure/wec_start_service.ps1 b/testing/configure/wec_start_service.ps1 new file mode 100644 index 00000000..7677185c --- /dev/null +++ b/testing/configure/wec_start_service.ps1 @@ -0,0 +1,19 @@ +# Start WEC using custom wec xml file + +try { + Start-Service -Name "Wecsvc" + Write-Output "WEC service started successfully." +} catch { + Write-Output "Failed to start WEC service: $_" +} + +$ConfigFilePath = "$env:USERPROFILE\Downloads\LME\Chapter 1 Files\lme_wec_config.xml" + +try { + Start-Process -FilePath "wecutil.exe" -ArgumentList "cs `"$ConfigFilePath`"" -Verb RunAs + Write-Output "wecutil command executed successfully." +} catch { + Write-Output "Failed to execute wecutil command: $_" +} + + diff --git a/testing/configure/winlogbeat_install.ps1 b/testing/configure/winlogbeat_install.ps1 new file mode 100644 index 00000000..7e78566c --- /dev/null +++ b/testing/configure/winlogbeat_install.ps1 @@ -0,0 +1,84 @@ +param ( + [Parameter()] + [string]$BaseDirectory = "C:\lme", + + [Parameter()] + [string]$WinlogbeatVersion = "winlogbeat-8.5.0-windows-x86_64" +) + +# Source and destination directories +$SourceDir = "$BaseDirectory\files_for_windows\tmp" +$DestinationDir = "C:\Program Files" + +# Copying files from source to destination +Copy-Item -Path "$SourceDir\*" -Destination $DestinationDir -Recurse -Force + +# Winlogbeat url +$Url = "https://artifacts.elastic.co/downloads/beats/winlogbeat/$WinlogbeatVersion.zip" + +# Destination path where the file will be saved +$WinlogbeatDestination = "$BaseDirectory\$WinlogbeatVersion.zip" + +# Unzip destination +$UnzipDestination = "C:\Program Files\lme\$WinlogbeatVersion" + +# Define the path of the winlogbeat.yml file in C:\Program Files\lme +$WinlogbeatYmlSource = "C:\Program Files\lme\winlogbeat.yml" + +# Define the destination path of the winlogbeat.yml file +$WinlogbeatYmlDestination = Join-Path -Path $UnzipDestination -ChildPath "winlogbeat.yml" + +# Define the full path of the install script +$InstallScriptPath = Join-Path -Path $UnzipDestination -ChildPath "install-service-winlogbeat.ps1" + +# Create the base directory if it does not exist +if (-not (Test-Path $BaseDirectory)) { + New-Item -ItemType Directory -Path $BaseDirectory +} + +# Download the file +Invoke-WebRequest -Uri $Url -OutFile $WinlogbeatDestination + +# Unzip the file +Expand-Archive -LiteralPath $WinlogbeatDestination -DestinationPath $UnzipDestination + +# Define the nested directory path +$nestedDir = Join-Path -Path $UnzipDestination -ChildPath $WinlogbeatVersion + +# Move the contents of the nested directory up one level and remove the nested directory +if (Test-Path $nestedDir) { + Get-ChildItem -Path $nestedDir -Recurse | Move-Item -Destination $UnzipDestination + Remove-Item -Path $nestedDir -Force -Recurse +} + +# Move the winlogbeat.yml file to the destination directory, overwriting if it exists +Move-Item -Path $WinlogbeatYmlSource -Destination $WinlogbeatYmlDestination -Force + +# Set execution policy to Unrestricted for this process +Set-ExecutionPolicy Unrestricted -Scope Process + +# Check if the install script exists +if (Test-Path $InstallScriptPath) { + # Change directory to the unzip destination + Push-Location -Path $UnzipDestination + + # Run the install script + .\install-service-winlogbeat.ps1 + + # Return to the previous directory + Pop-Location +} +else { + Write-Output "The installation script was not found at $InstallScriptPath" +} + +Start-Sleep -Seconds 5 + +# Start the winlogbeat service +try { + Start-Service -Name "winlogbeat" + Write-Output "Winlogbeat service started successfully." +} +catch { + Write-Output "Failed to start Winlogbeat service: $_" +}