Skip to content

Instantly share code, notes, and snippets.

@nullbus
Last active February 12, 2020 08:39
Show Gist options
  • Save nullbus/684761cefb6702425c1928031b4ffd2e to your computer and use it in GitHub Desktop.
Save nullbus/684761cefb6702425c1928031b4ffd2e to your computer and use it in GitHub Desktop.
modification of https://github.com/liamkf/Unreal_FASTBuild to make work on 4.22
// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.
/**
* This program is designed to execute the Visual C++ compiler (cl.exe) and filter off any output lines from the /showIncludes directive
* into a separate file for dependency checking. GCC/Clang have a specific option for this, whereas MSVC does not.
*/
#include <windows.h>
#include <stdio.h>
#include <string>
#include <vector>
#include <set>
void GetLocalizedIncludePrefixes(const wchar_t* CompilerPath, std::vector<std::vector<char>>& Prefixes);
DWORD WINAPI RedirectToNull(PVOID Params)
{
HANDLE StdOutReadHandle = *(HANDLE*)reinterpret_cast<void**>(Params)[0];
bool bIgnoreStdout = *(bool*)reinterpret_cast<void**>(Params)[1];
char Buffer[1024];
while (true)
{
DWORD BytesRead;
if (ReadFile(StdOutReadHandle, Buffer, sizeof(Buffer), &BytesRead, NULL))
{
if (!bIgnoreStdout)
{
DWORD BytesWritten;
WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), Buffer, BytesRead, &BytesWritten, NULL);
}
}
else if(GetLastError() != ERROR_BROKEN_PIPE)
{
wprintf(L"ERROR: Unable to read data from child process (%08x)", GetLastError());
}
else
{
break;
}
}
return 0;
}
int wmain(int ArgC, const wchar_t* ArgV[])
{
// Get the full command line, and find the '--' marker
wchar_t* CmdLine = ::GetCommandLineW();
wchar_t *ChildCmdLine = wcsstr(CmdLine, L" -- ");
if (ChildCmdLine == nullptr)
{
wprintf(L"ERROR: Unable to find child command line (%s)", CmdLine);
return -1;
}
ChildCmdLine += 4;
wchar_t* IgnoreStdout = wcsstr(CmdLine, L"--nostdout");
bool bIgnoreStdout = IgnoreStdout && IgnoreStdout < ChildCmdLine;
int ArgOffset = bIgnoreStdout ? 1 : 0;
// Make sure we've got an output file and compiler path
if (ArgC <= (4 + ArgOffset) || wcscmp(ArgV[2+ArgOffset], L"--") != 0)
{
wprintf(L"ERROR: Syntax: cl-filter <dependencies-file> -- <child command line>\n");
return -1;
}
// Get the arguments we care about
const wchar_t* OutputFileName = ArgV[1+ArgOffset];
const wchar_t* CompilerFileName = ArgV[3+ArgOffset];
// Get all the possible localized string prefixes for /showIncludes output
std::vector<std::vector<char>> LocalizedPrefixes;
GetLocalizedIncludePrefixes(CompilerFileName, LocalizedPrefixes);
// Create the child process
PROCESS_INFORMATION ProcessInfo;
ZeroMemory(&ProcessInfo, sizeof(ProcessInfo));
SECURITY_ATTRIBUTES SecurityAttributes;
ZeroMemory(&SecurityAttributes, sizeof(SecurityAttributes));
SecurityAttributes.bInheritHandle = TRUE;
HANDLE StdOutReadHandle;
HANDLE StdOutWriteHandle;
if (CreatePipe(&StdOutReadHandle, &StdOutWriteHandle, &SecurityAttributes, 0) == 0)
{
wprintf(L"ERROR: Unable to create output pipe for child process\n");
return -1;
}
HANDLE StdErrReadHandle;
HANDLE StdErrWriteHandle;
if (CreatePipe(&StdErrReadHandle, &StdErrWriteHandle, &SecurityAttributes, 0) == 0)
{
wprintf(L"ERROR: Unable to create stderr pipe handle for child process\n");
return -1;
}
void* ThreadArgs[] = { &StdOutReadHandle, &bIgnoreStdout };
HANDLE hNullRedirectThread = CreateThread(NULL, 0, RedirectToNull, ThreadArgs, 0, NULL);
// Create the new process as suspended, so we can modify it before it starts executing (and potentially preempting us)
STARTUPINFO StartupInfo;
ZeroMemory(&StartupInfo, sizeof(StartupInfo));
StartupInfo.cb = sizeof(StartupInfo);
StartupInfo.hStdInput = NULL;
StartupInfo.hStdOutput = StdOutWriteHandle;
StartupInfo.hStdError = StdErrWriteHandle;
StartupInfo.dwFlags = STARTF_USESTDHANDLES;
DWORD ProcessCreationFlags = GetPriorityClass(GetCurrentProcess());
if (CreateProcessW(NULL, ChildCmdLine, NULL, NULL, TRUE, ProcessCreationFlags, NULL, NULL, &StartupInfo, &ProcessInfo) == 0)
{
wprintf(L"ERROR: Unable to create child process\n");
return -1;
}
// Close the startup thread handle; we don't need it.
CloseHandle(ProcessInfo.hThread);
// Close the write ends of the handle. We don't want any other process to be able to inherit these.
CloseHandle(StdOutWriteHandle);
CloseHandle(StdErrWriteHandle);
// Delete the output file
DeleteFileW(OutputFileName);
// Get the path to a temporary output filename
std::wstring TempOutputFileName(OutputFileName);
TempOutputFileName += L".tmp";
// Create a file to contain the dependency list
HANDLE OutputFile = CreateFileW(TempOutputFileName.c_str(), GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if(OutputFile == INVALID_HANDLE_VALUE)
{
wprintf(L"ERROR: Unable to open %s for output", TempOutputFileName.c_str());
return -1;
}
// Pipe the output to stdout
char Buffer[1024];
size_t BufferSize = 0;
for (;;)
{
// Read the next chunk of data from the output stream
DWORD BytesRead = 0;
if (BufferSize < sizeof(Buffer))
{
if (ReadFile(StdErrReadHandle, Buffer + BufferSize, (DWORD)(sizeof(Buffer) - BufferSize), &BytesRead, NULL))
{
BufferSize += BytesRead;
}
else if(GetLastError() != ERROR_BROKEN_PIPE)
{
wprintf(L"ERROR: Unable to read data from child process (%08x)", GetLastError());
}
else if (BufferSize == 0)
{
break;
}
}
// Parse individual lines from the output
size_t LineStart = 0;
while(LineStart < BufferSize)
{
// Find the end of this line
size_t LineEnd = LineStart;
while (LineEnd < BufferSize && Buffer[LineEnd] != '\n')
{
LineEnd++;
}
// If we didn't reach a line terminator, and we can still read more data, clear up some space and try again
if (LineEnd == BufferSize && !(LineStart == 0 && BytesRead == 0) && !(LineStart == 0 && BufferSize == sizeof(Buffer)))
{
break;
}
// Skip past the EOL marker
if (LineEnd < BufferSize && Buffer[LineEnd] == '\n')
{
LineEnd++;
}
// Filter out any lines that have the "Note: including file: " prefix.
for (const std::vector<char>& LocalizedPrefix : LocalizedPrefixes)
{
if (memcmp(Buffer + LineStart, LocalizedPrefix.data(), LocalizedPrefix.size() - 1) == 0)
{
size_t FileNameStart = LineStart + LocalizedPrefix.size() - 1;
while (FileNameStart < LineEnd && isspace(Buffer[FileNameStart]))
{
FileNameStart++;
}
DWORD BytesWritten;
WriteFile(OutputFile, Buffer + FileNameStart, (DWORD)(LineEnd - FileNameStart), &BytesWritten, NULL);
LineStart = LineEnd;
break;
}
}
// If we didn't write anything out, write it to stdout
if(LineStart < LineEnd)
{
DWORD BytesWritten;
WriteFile(GetStdHandle(STD_OUTPUT_HANDLE), Buffer + LineStart, (DWORD)(LineEnd - LineStart), &BytesWritten, NULL);
}
// Move to the next line
LineStart = LineEnd;
}
// Shuffle everything down
memmove(Buffer, Buffer + LineStart, BufferSize - LineStart);
BufferSize -= LineStart;
}
WaitForSingleObject(ProcessInfo.hProcess, INFINITE);
WaitForSingleObject(hNullRedirectThread, INFINITE);
DWORD ExitCode;
if (!GetExitCodeProcess(ProcessInfo.hProcess, &ExitCode))
{
ExitCode = (DWORD)-1;
}
CloseHandle(OutputFile);
if (ExitCode == 0 && !MoveFileW(TempOutputFileName.c_str(), OutputFileName))
{
wprintf(L"ERROR: Unable to rename %s to %s\n", TempOutputFileName.c_str(), OutputFileName);
ExitCode = 1;
}
return ExitCode;
}
static std::string FindAndReplace(std::string Input, const std::string& FindStr, const std::string& ReplaceStr)
{
size_t Start = 0;
for (;;)
{
size_t Offset = Input.find(FindStr, Start);
if(Offset == std::wstring::npos)
{
break;
}
Input.replace(Offset, FindStr.size(), ReplaceStr);
Start = Offset + ReplaceStr.size();
}
return Input;
}
bool GetLocalizedIncludePrefix(UINT CodePage, const wchar_t* LibraryPath, HMODULE LibraryHandle, std::vector<char>& Prefix)
{
static const unsigned int ResourceId = 408;
// Read the string from the library
wchar_t Text[512];
if(LoadStringW(LibraryHandle, ResourceId, Text, sizeof(Text) / sizeof(Text[0])) == 0)
{
wprintf(L"WARNING: unable to read string %d from %s\n", ResourceId, LibraryPath);
return false;
}
// Find the end of the prefix
wchar_t* TextEnd = wcsstr(Text, L"%s%s");
if (TextEnd == nullptr)
{
wprintf(L"WARNING: unable to find substitution markers in format string '%s' (%s)", Text, LibraryPath);
return false;
}
// Figure out how large the buffer needs to be to hold the MBCS version
int Length = WideCharToMultiByte(CP_ACP, 0, Text, (int)(TextEnd - Text), NULL, 0, NULL, NULL);
if (Length == 0)
{
wprintf(L"WARNING: unable to query size for MBCS output buffer (input text '%s', library %s)", Text, LibraryPath);
return false;
}
// Resize the output buffer with space for a null terminator
Prefix.resize(Length + 1);
// Get the multibyte text
int Result = WideCharToMultiByte(CodePage, 0, Text, (int)(TextEnd - Text), Prefix.data(), Length, NULL, NULL);
if (Result == 0)
{
wprintf(L"WARNING: unable to get MBCS string (input text '%s', library %s)", Text, LibraryPath);
return false;
}
return true;
}
// Language packs for Visual Studio contain localized strings for the "Note: including file:" prefix we expect to see when running the compiler
// with the /showIncludes option. Enumerate all the possible languages that may be active, and build up an array of possible prefixes. We'll treat
// any of them as markers for included files.
void GetLocalizedIncludePrefixes(const wchar_t* CompilerPath, std::vector<std::vector<char>>& Prefixes)
{
// Get all the possible locale id's. Include en-us by default.
std::set<std::wstring> LocaleIds;
LocaleIds.insert(L"1033");
// The user default locale id
wchar_t LocaleIdString[20];
wsprintf(LocaleIdString, L"%d", GetUserDefaultLCID());
LocaleIds.insert(LocaleIdString);
// The system default locale id
wsprintf(LocaleIdString, L"%d", GetSystemDefaultLCID());
LocaleIds.insert(LocaleIdString);
// The Visual Studio locale setting
static const size_t VsLangMaxLen = 256;
wchar_t VsLangEnv[VsLangMaxLen];
if (GetEnvironmentVariableW(L"VSLANG", VsLangEnv, VsLangMaxLen) != 0)
{
LocaleIds.insert(VsLangEnv);
}
// Find the directory containing the compiler path
size_t CompilerDirLen = wcslen(CompilerPath);
while (CompilerDirLen > 0 && CompilerPath[CompilerDirLen - 1] != '/' && CompilerPath[CompilerDirLen - 1] != '\\')
{
CompilerDirLen--;
}
// Always add the en-us prefix. We'll validate that this is correct if we have an en-us resource file, but it gives us something to fall back on.
const char EnglishText[] = "Note: including file:";
Prefixes.emplace_back(EnglishText, strchr(EnglishText, 0) + 1);
// Get the default console codepage
UINT CodePage = GetConsoleOutputCP();
// Loop through all the possible locale ids and try to find the localized string for each
for (const std::wstring& LocaleId : LocaleIds)
{
std::wstring ResourceFile;
ResourceFile.assign(CompilerPath, CompilerPath + CompilerDirLen);
ResourceFile.append(LocaleId);
ResourceFile.append(L"\\clui.dll");
HMODULE LibraryHandle = LoadLibraryExW(ResourceFile.c_str(), 0, LOAD_LIBRARY_AS_DATAFILE | LOAD_LIBRARY_AS_IMAGE_RESOURCE);
if (LibraryHandle != nullptr)
{
std::vector<char> Prefix;
if (GetLocalizedIncludePrefix(CodePage, ResourceFile.c_str(), LibraryHandle, Prefix))
{
if (wcscmp(LocaleId.c_str(), L"1033") != 0)
{
Prefixes.push_back(std::move(Prefix));
}
else if(strcmp(Prefix.data(), EnglishText) != 0)
{
wprintf(L"WARNING: unexpected localized string for en-us.\n Expected: '%hs'\n Actual: '%hs'", FindAndReplace(EnglishText, "\n", "\\n").c_str(), FindAndReplace(Prefix.data(), "\n", "\\n").c_str());
}
}
FreeLibrary(LibraryHandle);
}
}
}
// Copyright 2018 Yassine Riahi and Liam Flookes. Provided under a MIT License, see license file on github.
// Used to generate a fastbuild .bff file from UnrealBuildTool to allow caching and distributed builds.
// Tested with Windows 10, Visual Studio 2015/2017, Unreal Engine 4.19.1, FastBuild v0.95
// Durango is fully supported (Compiles with VS2015).
// Orbis will likely require some changes.
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Diagnostics;
using System.Linq;
using Tools.DotNETCommon;
namespace UnrealBuildTool
{
class FASTBuild : ActionExecutor
{
/*---- Configurable User settings ----*/
// Used to specify a non-standard location for the FBuild.exe, for example if you have not added it to your PATH environment variable.
public static string FBuildExePathOverride = @"C:\FastBuild\FBuild.exe";
// Controls network build distribution
private bool bEnableDistribution = true;
// Controls whether to use caching at all. CachePath and CacheMode are only relevant if this is enabled.
private bool bEnableCaching = false;
// Location of the shared cache, it could be a local or network path (i.e: @"\\DESKTOP-BEAST\FASTBuildCache").
// Only relevant if bEnableCaching is true;
private string CachePath = @"\\SoccerBuild\FastBuildShared\Cache";
// temporary storage of deps response files list
private List<string> DepsResponseFiles = new List<string>();
public enum eCacheMode
{
ReadWrite, // This machine will both read and write to the cache
ReadOnly, // This machine will only read from the cache, use for developer machines when you have centralized build machines
WriteOnly, // This machine will only write from the cache, use for build machines when you have centralized build machines
}
// Cache access mode
// Only relevant if bEnableCaching is true;
private eCacheMode CacheMode = eCacheMode.ReadWrite;
/*--------------------------------------*/
public override string Name
{
get { return "FASTBuild"; }
}
public static bool IsAvailable()
{
if (FBuildExePathOverride != "")
{
return File.Exists(FBuildExePathOverride);
}
// Get the name of the FASTBuild executable.
string fbuild = "fbuild";
if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64)
{
fbuild = "fbuild.exe";
}
// Search the path for it
string PathVariable = Environment.GetEnvironmentVariable("PATH");
foreach (string SearchPath in PathVariable.Split(Path.PathSeparator))
{
try
{
string PotentialPath = Path.Combine(SearchPath, fbuild);
if (File.Exists(PotentialPath))
{
return true;
}
}
catch (ArgumentException)
{
// PATH variable may contain illegal characters; just ignore them.
}
}
return false;
}
private HashSet<string> ForceLocalCompileModules = new HashSet<string>()
{"Module.ProxyLODMeshReduction",
"GoogleVRController"};
private enum FBBuildType
{
Windows32,
Windows64,
XBOne,
PS4,
Android,
Linux,
}
private FBBuildType BuildType = FBBuildType.Windows64;
private void DetectBuildType(List<Action> Actions)
{
foreach (Action action in Actions)
{
if (action.ActionType != ActionType.Compile && action.ActionType != ActionType.Link)
continue;
if (action.CommandPath.FullName.Contains("orbis"))
{
BuildType = FBBuildType.PS4;
return;
}
else if (action.CommandArguments.Contains("Intermediate\\Build\\XboxOne"))
{
BuildType = FBBuildType.XBOne;
return;
}
else if (action.CommandPath.GetFileName() == "cl-filter.exe" || action.CommandPath.FullName.Contains("Microsoft")) //Not a great test.
{
if (action.CommandPath.FullName.Contains("x64") || action.CommandArguments.Contains("x64"))
{
BuildType = FBBuildType.Windows64;
}
else
{
BuildType = FBBuildType.Windows32;
}
return;
}
else if (action.CommandArguments.Contains("Linux"))
{
BuildType = FBBuildType.Linux;
}
else if (action.CommandArguments.Contains("Android"))
{
BuildType = FBBuildType.Android;
return;
}
}
}
private bool IsMSVC() { return BuildType == FBBuildType.Windows32 || BuildType == FBBuildType.Windows64 || BuildType == FBBuildType.XBOne; }
private bool IsPS4() { return BuildType == FBBuildType.PS4; }
private bool IsXBOnePDBUtil(Action action) { return action.CommandPath.GetFileName().Contains("XboxOnePDBFileUtil.exe"); }
private bool IsPS4SymbolTool(Action action) { return action.CommandPath.GetFileName().Contains("PS4SymbolTool.exe"); }
private string GetCompilerName()
{
switch (BuildType)
{
default:
case FBBuildType.XBOne:
case FBBuildType.Android:
case FBBuildType.Windows32:
case FBBuildType.Windows64: return "UE4Compiler";
case FBBuildType.PS4: return "UE4PS4Compiler";
}
}
//Run FASTBuild on the list of actions. Relies on fbuild.exe being in the path.
public override bool ExecuteActions(List<Action> Actions, bool bLogDetailedActionStats)
{
bool FASTBuildResult = true;
if (Actions.Count > 0)
{
DetectBuildType(Actions);
string FASTBuildFilePath = Path.Combine(UnrealBuildTool.EngineDirectory.FullName, "Intermediate", "Build", "fbuild.bff");
List<Action> LocalExecutorActions = new List<Action>();
int NumFastbuildActions = 0;
if (CreateBffFile(Actions, FASTBuildFilePath, LocalExecutorActions, out NumFastbuildActions))
{
if (NumFastbuildActions > 0)
{
FASTBuildResult = ExecuteBffFile(FASTBuildFilePath);
}
if (FASTBuildResult)
{
LocalExecutor localExecutor = new LocalExecutor();
FASTBuildResult = localExecutor.ExecuteActions(LocalExecutorActions, bLogDetailedActionStats);
}
}
else
{
FASTBuildResult = false;
}
}
return FASTBuildResult;
}
private void AddText(string StringToWrite)
{
byte[] Info = new System.Text.UTF8Encoding(true).GetBytes(StringToWrite);
bffOutputFileStream.Write(Info, 0, Info.Length);
}
private string SubstituteEnvironmentVariables(string commandLineString)
{
string outputString = commandLineString.Replace("$(DurangoXDK)", "$DurangoXDK$");
outputString = outputString.Replace("$(SCE_ORBIS_SDK_DIR)", "$SCE_ORBIS_SDK_DIR$");
outputString = outputString.Replace("$(DXSDK_DIR)", "$DXSDK_DIR$");
outputString = outputString.Replace("$(CommonProgramFiles)", "$CommonProgramFiles$");
return outputString;
}
private List<string> RawTokensToParameters(IEnumerable<string> RawTokens)
{
// Raw tokens being split with spaces may have split up some two argument options and
// paths with multiple spaces in them also need some love
List<string> ProcessedTokens = new List<string>();
bool QuotesOpened = false;
string PartialToken = "";
IEnumerator<string> iter = RawTokens.GetEnumerator();
while (iter.MoveNext())
{
string Token = iter.Current;
if (string.IsNullOrEmpty(Token))
{
if (ProcessedTokens.Count > 0 && QuotesOpened)
{
string CurrentToken = ProcessedTokens.Last();
CurrentToken += " ";
}
continue;
}
int numQuotes = 0;
// Look for unescaped " symbols, we want to stick those strings into one token.
for (int j = 0; j < Token.Length; ++j)
{
if (Token[j] == '\\') //Ignore escaped quotes
++j;
else if (Token[j] == '"')
numQuotes++;
}
// Defines can have escaped quotes and other strings inside them
// so we consume tokens until we've closed any open unescaped parentheses.
if ((Token.StartsWith("/D") || Token.StartsWith("-D")) && !QuotesOpened)
{
if (numQuotes == 0 || numQuotes == 2)
{
ProcessedTokens.Add(Token);
}
else
{
PartialToken = Token;
bool AddedToken = false;
while (iter.MoveNext())
{
string NextToken = iter.Current;
if (string.IsNullOrEmpty(NextToken))
{
PartialToken += " ";
}
else if (!NextToken.EndsWith("\\\"") && NextToken.EndsWith("\"")) //Looking for a token that ends with a non-escaped "
{
ProcessedTokens.Add(PartialToken + " " + NextToken);
AddedToken = true;
break;
}
else
{
PartialToken += " " + NextToken;
}
}
if (!AddedToken)
{
Console.WriteLine("Warning! Looks like an unterminated string in tokens. Adding PartialToken and hoping for the best. Command line: " + string.Join(" ", RawTokens));
ProcessedTokens.Add(PartialToken);
}
}
continue;
}
if (!QuotesOpened)
{
if (numQuotes % 2 != 0) //Odd number of quotes in this token
{
PartialToken = Token + " ";
QuotesOpened = true;
}
else
{
ProcessedTokens.Add(Token);
}
}
else
{
if (numQuotes % 2 != 0) //Odd number of quotes in this token
{
ProcessedTokens.Add(PartialToken + Token);
QuotesOpened = false;
}
else
{
PartialToken += Token + " ";
}
}
}
return ProcessedTokens;
}
private Dictionary<string, string> ParseCommandLineOptions(string CompilerCommandLine, string[] SpecialOptions, bool SaveResponseFile = false, bool SkipInputFile = false)
{
Dictionary<string, string> ParsedCompilerOptions = new Dictionary<string, string>();
// Make sure we substituted the known environment variables with corresponding BFF friendly imported vars
CompilerCommandLine = SubstituteEnvironmentVariables(CompilerCommandLine);
// Some tricky defines /DTROUBLE=\"\\\" abc 123\\\"\" aren't handled properly by either Unreal or Fastbuild, but we do our best.
char[] SpaceChar = { ' ' };
List<string> RawTokens = RawTokensToParameters(CompilerCommandLine.Trim().Split(' '));
List<string> ProcessedTokens = new List<string>();
string ResponseFilePath = "";
int ResponseFileTokenIdx = -1;
for (int i = 0; i < RawTokens.Count; ++i)
{
if (RawTokens[i].StartsWith("@\""))
{
ResponseFileTokenIdx = i;
ResponseFilePath = RawTokens[i].Substring(2, RawTokens[i].Length - 3); // bit of a bodge to get the @"response.txt" path...
break;
}
}
//if (RawTokens.Length >= 1 && RawTokens[0].StartsWith("@\"")) //Response files are in 4.13 by default. Changing VCToolChain to not do this is probably better.
if (ResponseFileTokenIdx >= 0)
{
try
{
string ResponseFileText = File.ReadAllText(ResponseFilePath);
// Make sure we substituted the known environment variables with corresponding BFF friendly imported vars
ResponseFileText = SubstituteEnvironmentVariables(ResponseFileText);
string[] Separators = { "\n", " ", "\r" };
if (File.Exists(ResponseFilePath))
{
RawTokens.RemoveAt(ResponseFileTokenIdx);
RawTokens.InsertRange(
ResponseFileTokenIdx,
RawTokensToParameters(ResponseFileText.Split(Separators, StringSplitOptions.RemoveEmptyEntries))
); //Certainly not ideal
}
}
catch (Exception e)
{
Console.WriteLine("Looks like a response file in: " + CompilerCommandLine + ", but we could not load it! " + e.Message);
ResponseFilePath = "";
}
}
ProcessedTokens = RawTokens;
//Processed tokens should now have 'whole' tokens, so now we look for any specified special options
foreach (string specialOption in SpecialOptions)
{
for (int i = 0; i < ProcessedTokens.Count; ++i)
{
if (ProcessedTokens[i] == specialOption && i + 1 < ProcessedTokens.Count)
{
ParsedCompilerOptions[specialOption] = ProcessedTokens[i + 1];
ProcessedTokens.RemoveRange(i, 2);
break;
}
else if (ProcessedTokens[i].StartsWith(specialOption))
{
ParsedCompilerOptions[specialOption] = ProcessedTokens[i].Replace(specialOption, null);
ProcessedTokens.RemoveAt(i);
break;
}
}
}
//The search for the input file... we take the first non-argument we can find
if (!SkipInputFile)
{
HashSet<string> TokensNeedArgument = new HashSet<string> {
"/I", "-I", "/l", "/D", "-D", "-x", "-include", "-std", "-march",
"-target", "--sysroot", "-gcc-toolchain", "-fdiagnostics-format", "-isystem"
};
for (int i = 0; i < ProcessedTokens.Count; ++i)
{
string Token = ProcessedTokens[i];
if (Token.Length == 0)
{
continue;
}
if (TokensNeedArgument.Contains(Token)) // Skip tokens with values, I for cpp includes, l for resource compiler includes
{
if (Token == "/I" || Token == "-I")
{
string Argument = ProcessedTokens[i + 1];
if (Argument.First() == '"' && Argument.Last() == '"')
{
ProcessedTokens[i] = string.Format("{0}{1}", Token, ProcessedTokens[i + 1]);
}
else
{
ProcessedTokens[i] = string.Format("{0}\"{1}\"", Token, ProcessedTokens[i + 1]);
}
ProcessedTokens.RemoveAt(i + 1);
}
else
{
++i;
}
}
else if (!Token.StartsWith("/") && !Token.StartsWith("-") && !Token.StartsWith("\"-"))
{
ParsedCompilerOptions["InputFile"] = Token;
ProcessedTokens.RemoveAt(i);
break;
}
}
}
ParsedCompilerOptions["OtherOptions"] = string.Join(" ", ProcessedTokens) + " ";
if (SaveResponseFile && !string.IsNullOrEmpty(ResponseFilePath))
{
ParsedCompilerOptions["@"] = ResponseFilePath;
}
return ParsedCompilerOptions;
}
private List<Action> SortActions(List<Action> InActions)
{
List<Action> Actions = InActions;
int NumSortErrors = 0;
for (int ActionIndex = 0; ActionIndex < InActions.Count; ActionIndex++)
{
Action Action = InActions[ActionIndex];
foreach (Action RequiredAction in Action.PrerequisiteActions)
{
if (InActions.Contains(RequiredAction))
{
int DepIndex = InActions.IndexOf(RequiredAction);
if (DepIndex > ActionIndex)
{
NumSortErrors++;
}
}
}
}
if (NumSortErrors > 0)
{
Actions = new List<Action>();
var UsedActions = new HashSet<int>();
for (int ActionIndex = 0; ActionIndex < InActions.Count; ActionIndex++)
{
if (UsedActions.Contains(ActionIndex))
{
continue;
}
Action Action = InActions[ActionIndex];
foreach (Action RequiredAction in Action.PrerequisiteActions)
{
if (InActions.Contains(RequiredAction))
{
int DepIndex = InActions.IndexOf(RequiredAction);
if (UsedActions.Contains(DepIndex))
{
continue;
}
Actions.Add(RequiredAction);
UsedActions.Add(DepIndex);
}
}
Actions.Add(Action);
UsedActions.Add(ActionIndex);
}
for (int ActionIndex = 0; ActionIndex < Actions.Count; ActionIndex++)
{
Action Action = Actions[ActionIndex];
foreach (Action RequiredAction in Action.PrerequisiteActions)
{
if (Actions.Contains(RequiredAction))
{
int DepIndex = Actions.IndexOf(RequiredAction);
if (DepIndex > ActionIndex)
{
Console.WriteLine("Action is not topologically sorted.");
Console.WriteLine(" {0} {1}", Action.CommandPath, Action.CommandArguments);
Console.WriteLine("Dependency");
Console.WriteLine(" {0} {1}", RequiredAction.CommandPath, RequiredAction.CommandArguments);
throw new BuildException("Cyclical Dependency in action graph.");
}
}
}
}
}
return Actions;
}
private string GetOptionValue(Dictionary<string, string> OptionsDictionary, string Key, Action Action, bool ProblemIfNotFound = false)
{
string Value = string.Empty;
if (OptionsDictionary.TryGetValue(Key, out Value))
{
return Value.Trim(new Char[] { '\"' });
}
if (ProblemIfNotFound)
{
Console.WriteLine("We failed to find " + Key + ", which may be a problem.");
Console.WriteLine("Action.CommandArguments: " + Action.CommandArguments);
}
return Value;
}
public string GetRegistryValue(string keyName, string valueName, object defaultValue)
{
object returnValue = (string)Microsoft.Win32.Registry.GetValue("HKEY_LOCAL_MACHINE\\SOFTWARE\\" + keyName, valueName, defaultValue);
if (returnValue != null)
return returnValue.ToString();
returnValue = Microsoft.Win32.Registry.GetValue("HKEY_CURRENT_USER\\SOFTWARE\\" + keyName, valueName, defaultValue);
if (returnValue != null)
return returnValue.ToString();
returnValue = (string)Microsoft.Win32.Registry.GetValue("HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\" + keyName, valueName, defaultValue);
if (returnValue != null)
return returnValue.ToString();
returnValue = Microsoft.Win32.Registry.GetValue("HKEY_CURRENT_USER\\SOFTWARE\\Wow6432Node\\" + keyName, valueName, defaultValue);
if (returnValue != null)
return returnValue.ToString();
return defaultValue.ToString();
}
private void WriteEnvironmentSetup(List<Action> Actions)
{
DirectoryReference VCInstallDir = null;
string VCToolPath64 = "";
VCEnvironment VCEnv = null;
try
{
// This may fail if the caller emptied PATH; we try to ignore the problem since
// it probably means we are building for another platform.
if (BuildType == FBBuildType.Windows64)
{
VCEnv = VCEnvironment.Create(WindowsPlatform.GetDefaultCompiler(null), CppPlatform.Win64, null, null);
}
else if (BuildType == FBBuildType.Windows32)
{
VCEnv = VCEnvironment.Create(WindowsPlatform.GetDefaultCompiler(null), CppPlatform.Win32, null, null);
}
else if (BuildType == FBBuildType.XBOne)
{
// If you have XboxOne source access, uncommenting the line below will be better for selecting the appropriate version of the compiler.
// Translate the XboxOne compiler to the right Windows compiler to set the VC environment vars correctly...
//WindowsCompiler windowsCompiler = XboxOnePlatform.GetDefaultCompiler() == XboxOneCompiler.VisualStudio2015 ? WindowsCompiler.VisualStudio2015 : WindowsCompiler.VisualStudio2017;
//VCEnv = VCEnvironment.Create(windowsCompiler, CppPlatform.Win64, null, null);
}
}
catch (Exception)
{
Console.WriteLine("Failed to get Visual Studio environment.");
}
// Copy environment into a case-insensitive dictionary for easier key lookups
Dictionary<string, string> envVars = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (DictionaryEntry entry in Environment.GetEnvironmentVariables())
{
envVars[(string)entry.Key] = (string)entry.Value;
}
if (envVars.ContainsKey("CommonProgramFiles"))
{
AddText("#import CommonProgramFiles\n");
}
if (envVars.ContainsKey("DXSDK_DIR"))
{
AddText("#import DXSDK_DIR\n");
}
if (envVars.ContainsKey("DurangoXDK"))
{
AddText("#import DurangoXDK\n");
}
if (VCEnv != null)
{
string platformVersionNumber = "VSVersionUnknown";
string platformVersionNumber2 = "VSVersionUnknown";
switch (VCEnv.Compiler)
{
case WindowsCompiler.VisualStudio2015:
platformVersionNumber = "140";
break;
case WindowsCompiler.VisualStudio2017:
// For now we are working with the 140 version, might need to change to 141 or 150 depending on the version of the Toolchain you chose
// to install
platformVersionNumber = "140";
platformVersionNumber2 = "141";
break;
case WindowsCompiler.VisualStudio2019:
platformVersionNumber = "140";
platformVersionNumber2 = "142";
break;
default:
string exceptionString = "Error: Unsupported Visual Studio Version.";
Console.WriteLine(exceptionString);
throw new BuildException(exceptionString);
}
if (!WindowsPlatform.TryGetVSInstallDir(WindowsPlatform.GetDefaultCompiler(null), out VCInstallDir))
{
string exceptionString = "Error: Cannot locate Visual Studio Installation.";
Console.WriteLine(exceptionString);
throw new BuildException(exceptionString);
}
VCToolPath64 = VCEnvironment.GetVCToolPath64(WindowsPlatform.GetDefaultCompiler(null), VCEnv.ToolChainDir).ToString();
string debugVCToolPath64 = VCEnv.CompilerPath.Directory.ToString();
AddText(string.Format(".WindowsSDKBasePath = '{0}'\n", VCEnv.WindowsSdkDir));
AddText("Compiler('UE4ResourceCompiler') \n{\n");
AddText(string.Format("\t.Executable = '{0}'\n", VCEnv.ResourceCompilerPath));
AddText("\t.CompilerFamily = 'custom'\n");
AddText("}\n\n");
AddText("Compiler('UE4DependencyCompiler') \n{\n");
AddText(string.Format("\t.Executable = '{0}'\n", FileReference.Combine(UnrealBuildTool.EngineDirectory, "Build", "Windows", "cl-filter", "cl-filter.exe")));
AddText("\t.CompilerFamily = 'custom'\n");
AddText("\t.AllowDistribution = false\n");
AddText("\t.AllowCaching = false\n");
AddText("}\n\n");
AddText("Compiler('UE4Compiler') \n{\n");
if (BuildType == FBBuildType.Windows64)
{
AddText(string.Format("\t.Root = '{0}'\n", VCEnv.CompilerPath.Directory));
}
else
{
DirectoryReference X86ToolPath = DirectoryReference.Combine(VCEnv.CompilerPath.Directory, "../../Hostx86/x86");
AddText(string.Format("\t.Root = '{0}'\n", X86ToolPath));
}
AddText("\t.Executable = '$Root$/cl.exe'\n");
AddText("\t.ExtraFiles =\n\t{\n");
AddText("\t\t'$Root$/c1.dll'\n");
AddText("\t\t'$Root$/c1xx.dll'\n");
AddText("\t\t'$Root$/c2.dll'\n");
if (File.Exists(FileReference.Combine(VCEnv.CompilerPath.Directory, "1033/clui.dll").ToString())) //Check English first...
{
AddText("\t\t'$Root$/1033/clui.dll'\n");
}
else
{
var numericDirectories = Directory.GetDirectories(VCToolPath64).Where(d => Path.GetFileName(d).All(char.IsDigit));
var cluiDirectories = numericDirectories.Where(d => Directory.GetFiles(d, "clui.dll").Any());
if (cluiDirectories.Any())
{
AddText(string.Format("\t\t'$Root$/{0}/clui.dll'\n", Path.GetFileName(cluiDirectories.First())));
}
}
AddText("\t\t'$Root$/mspdbsrv.exe'\n");
AddText("\t\t'$Root$/mspdbcore.dll'\n");
AddText(string.Format("\t\t'$Root$/mspft{0}.dll'\n", platformVersionNumber));
AddText(string.Format("\t\t'$Root$/msobj{0}.dll'\n", platformVersionNumber));
AddText(string.Format("\t\t'$Root$/mspdb{0}.dll'\n", platformVersionNumber));
if (VCEnv.Compiler == WindowsCompiler.VisualStudio2015)
{
AddText(string.Format("\t\t'{0}/VC/redist/x64/Microsoft.VC{1}.CRT/msvcp{2}.dll'\n", VCInstallDir.ToString(), platformVersionNumber, platformVersionNumber));
AddText(string.Format("\t\t'{0}/VC/redist/x64/Microsoft.VC{1}.CRT/vccorlib{2}.dll'\n", VCInstallDir.ToString(), platformVersionNumber, platformVersionNumber));
}
else
{
//VS 2017 is really confusing in terms of version numbers and paths so these values might need to be modified depending on what version of the tool chain you
// chose to install.
string RedistFilePath = string.Format("{0}/VC/Auxiliary/Build/Microsoft.VCRedistVersion.default.txt", VCInstallDir);
string RedistVersionNumber = File.ReadAllText(RedistFilePath).TrimEnd('\r', '\n');
string Architecture = BuildType == FBBuildType.Windows32 ? "x86" : "x64";
AddText(string.Format("\t\t'{0}/VC/Redist/MSVC/{1}/{2}/Microsoft.VC{3}.CRT/msvcp{4}.dll'\n", VCInstallDir.ToString(), RedistVersionNumber, Architecture, platformVersionNumber2, platformVersionNumber));
AddText(string.Format("\t\t'{0}/VC/Redist/MSVC/{1}/{2}/Microsoft.VC{3}.CRT/vccorlib{4}.dll'\n", VCInstallDir.ToString(), RedistVersionNumber, Architecture, platformVersionNumber2, platformVersionNumber));
}
AddText("\t}\n"); //End extra files
AddText("}\n\n"); //End compiler
}
else if (BuildType == FBBuildType.Android || BuildType == FBBuildType.Linux)
{
FileReference CompilerPath = null;
foreach (Action action in Actions)
{
if (action.ActionType == ActionType.Compile || action.ActionType == ActionType.Link)
{
CompilerPath = action.CommandPath;
break;
}
}
if (CompilerPath == null)
{
throw new BuildException("cannot detect clang compiler path");
}
AddText("Compiler('UE4Compiler') \n{\n");
AddText(string.Format("\t.Root = '{0}'\n", CompilerPath.Directory));
if (CompilerPath.GetExtension() == "")
{
AddText(string.Format("\t.Executable = '$Root$/{0}.exe'\n", CompilerPath.GetFileName()));
}
else
{
AddText(string.Format("\t.Executable = '$Root$/{0}'\n", CompilerPath.GetFileName()));
}
if (CompilerPath.FullName.Contains("clang++"))
{
AddText("\t.ClangRewriteIncludes = false\n");
}
AddText("}\n\n");
}
if (envVars.ContainsKey("SCE_ORBIS_SDK_DIR"))
{
AddText(string.Format(".SCE_ORBIS_SDK_DIR = '{0}'\n", envVars["SCE_ORBIS_SDK_DIR"]));
AddText(string.Format(".PS4BasePath = '{0}/host_tools/bin'\n\n", envVars["SCE_ORBIS_SDK_DIR"]));
AddText("Compiler('UE4PS4Compiler') \n{\n");
AddText("\t.Executable = '$PS4BasePath$/orbis-clang.exe'\n");
AddText("\t.ExtraFiles = '$PS4BasePath$/orbis-snarl.exe'\n");
AddText("}\n\n");
}
AddText("Settings \n{\n");
// Optional cachePath user setting
if (bEnableCaching && CachePath != "")
{
AddText(string.Format("\t.CachePath = '{0}'\n", CachePath));
}
//Start Environment
AddText("\t.Environment = \n\t{\n");
if (VCEnv != null)
{
AddText(string.Format("\t\t\"PATH={0}\\Common7\\IDE\\;{1}\",\n", VCInstallDir.ToString(), VCToolPath64));
if (VCEnv.IncludePaths.Count() > 0)
{
AddText(string.Format("\t\t\"INCLUDE={0}\",\n", String.Join(";", VCEnv.IncludePaths.Select(x => x))));
}
if (VCEnv.LibraryPaths.Count() > 0)
{
AddText(string.Format("\t\t\"LIB={0}\",\n", String.Join(";", VCEnv.LibraryPaths.Select(x => x))));
}
}
if (envVars.ContainsKey("TMP"))
AddText(string.Format("\t\t\"TMP={0}\",\n", envVars["TMP"]));
if (envVars.ContainsKey("SystemRoot"))
AddText(string.Format("\t\t\"SystemRoot={0}\",\n", envVars["SystemRoot"]));
if (envVars.ContainsKey("INCLUDE"))
AddText(string.Format("\t\t\"INCLUDE={0}\",\n", envVars["INCLUDE"]));
if (envVars.ContainsKey("LIB"))
AddText(string.Format("\t\t\"LIB={0}\",\n", envVars["LIB"]));
AddText("\t}\n"); //End environment
AddText("}\n\n"); //End Settings
}
private void AddCompileAction(Action Action, int ActionIndex, List<int> DependencyIndices)
{
bool bUseClFilter = Action.CommandPath.GetFileName() == "cl-filter.exe";
string CompilerName = GetCompilerName();
string CommandArguments = Action.CommandArguments;
string DependencySubExec = "";
string DependencyFile = "";
if (Action.CommandPath.GetFileName() == "rc.exe")
{
CompilerName = "UE4ResourceCompiler";
}
else if (bUseClFilter)
{
// cl-filter.exe "dep-path" -- subcommand [args...]
List<string> ParsedArguments = RawTokensToParameters(Action.CommandArguments.Split(' '));
// dependency info
DependencyFile = ParsedArguments[0];
DependencySubExec = ParsedArguments[2];
// skip [[dep-path] [--] [subcommand]]
CommandArguments = string.Join(" ", ParsedArguments.Skip(3));
}
string[] SpecialCompilerOptions = { "/Fo", "/fo", "/Yc", "/Yu", "/Fp", "-o" };
var ParsedCompilerOptions = ParseCommandLineOptions(CommandArguments, SpecialCompilerOptions);
string OutputObjectFileName = GetOptionValue(ParsedCompilerOptions, IsMSVC() ? "/Fo" : "-o", Action, ProblemIfNotFound: !IsMSVC());
if (IsMSVC() && string.IsNullOrEmpty(OutputObjectFileName)) // Didn't find /Fo, try /fo
{
OutputObjectFileName = GetOptionValue(ParsedCompilerOptions, "/fo", Action, ProblemIfNotFound: true);
}
if (string.IsNullOrEmpty(OutputObjectFileName)) //No /Fo or /fo, we're probably in trouble.
{
Console.WriteLine("We have no OutputObjectFileName. Bailing.");
return;
}
string IntermediatePath = Path.GetDirectoryName(OutputObjectFileName);
if (string.IsNullOrEmpty(IntermediatePath))
{
Console.WriteLine("We have no IntermediatePath. Bailing.");
Console.WriteLine("Our Action.CommandArguments were: " + CommandArguments);
return;
}
string InputFile = GetOptionValue(ParsedCompilerOptions, "InputFile", Action, ProblemIfNotFound: true);
if (string.IsNullOrEmpty(InputFile))
{
Console.WriteLine("We have no InputFile. Bailing.");
return;
}
string OtherCompilerOptions = GetOptionValue(ParsedCompilerOptions, "OtherOptions", Action);
AddText(string.Format("ObjectList('Action_{0}')\n{{\n", ActionIndex));
AddText(string.Format("\t.Compiler = '{0}' \n", CompilerName));
AddText(string.Format("\t.CompilerInputFiles = \"{0}\"\n", InputFile));
AddText(string.Format("\t.CompilerOutputPath = \"{0}\"\n", IntermediatePath));
bool bSkipDistribution = false;
foreach (var it in ForceLocalCompileModules)
{
try
{
if (Path.GetFullPath(InputFile).Contains(it))
{
bSkipDistribution = true;
break;
}
}
catch (NotSupportedException e)
{ }
}
if (!Action.bCanExecuteRemotely || !Action.bCanExecuteRemotelyWithSNDBS || bSkipDistribution)
{
AddText(string.Format("\t.AllowDistribution = false\n"));
}
string CompilerOutputExtension = ".unset";
if (BuildType == FBBuildType.Linux)
{
// Distcc separates preprocessing and compilation which means we must silence these warnings
OtherCompilerOptions += " -Wno-parentheses-equality";
OtherCompilerOptions += " -Wno-constant-logical-operand";
}
if (ParsedCompilerOptions.ContainsKey("/Yc")) //Create PCH
{
string PCHIncludeHeader = GetOptionValue(ParsedCompilerOptions, "/Yc", Action, ProblemIfNotFound: true);
string PCHOutputFile = GetOptionValue(ParsedCompilerOptions, "/Fp", Action, ProblemIfNotFound: true);
AddText(string.Format("\t.CompilerOptions = '\"%1\" /Fo\"%2\" /Fp\"{0}\" /Yu\"{1}\" {2} '\n", PCHOutputFile, PCHIncludeHeader, OtherCompilerOptions));
AddText(string.Format("\t.PCHOptions = '\"%1\" /Fp\"%2\" /Yc\"{0}\" {1} /Fo\"{2}\"'\n", PCHIncludeHeader, OtherCompilerOptions, OutputObjectFileName));
AddText(string.Format("\t.PCHInputFile = \"{0}\"\n", InputFile));
AddText(string.Format("\t.PCHOutputFile = \"{0}\"\n", PCHOutputFile));
CompilerOutputExtension = ".obj";
}
else if (ParsedCompilerOptions.ContainsKey("/Yu")) //Use PCH
{
string PCHIncludeHeader = GetOptionValue(ParsedCompilerOptions, "/Yu", Action, ProblemIfNotFound: true);
string PCHOutputFile = GetOptionValue(ParsedCompilerOptions, "/Fp", Action, ProblemIfNotFound: true);
string PCHToForceInclude = PCHOutputFile.Replace(".pch", "");
AddText(string.Format("\t.CompilerOptions = '\"%1\" /Fo\"%2\" /Fp\"{0}\" /Yu\"{1}\" /FI\"{2}\" {3} '\n", PCHOutputFile, PCHIncludeHeader, PCHToForceInclude, OtherCompilerOptions));
if (OutputObjectFileName.EndsWith("c.obj"))
{
CompilerOutputExtension = ".c.obj";
}
else
{
CompilerOutputExtension = ".cpp.obj";
}
}
else
{
if (CompilerName == "UE4ResourceCompiler")
{
AddText(string.Format("\t.CompilerOptions = '{0} /fo\"%2\" \"%1\" '\n", OtherCompilerOptions));
CompilerOutputExtension = Path.GetExtension(InputFile) + ".res";
}
else
{
if (IsMSVC())
{
AddText(string.Format("\t.CompilerOptions = '{0} /Fo\"%2\" \"%1\" '\n", OtherCompilerOptions));
if (OutputObjectFileName.EndsWith("c.obj"))
{
CompilerOutputExtension = ".c.obj";
}
else
{
CompilerOutputExtension = ".cpp.obj";
}
}
else
{
AddText(string.Format("\t.CompilerOptions = '{0} -o \"%2\" \"%1\" '\n", OtherCompilerOptions.Replace('\'', '"')));
int extensionIndex = OutputObjectFileName.LastIndexOf(".cpp");
if (extensionIndex < 0)
{
extensionIndex = OutputObjectFileName.LastIndexOf(".c");
}
CompilerOutputExtension = OutputObjectFileName.Substring(extensionIndex);
}
}
}
AddText(string.Format("\t.CompilerOutputExtension = '{0}' \n", CompilerOutputExtension));
if (DependencyIndices.Count > 0)
{
List<string> DependencyNames = DependencyIndices.ConvertAll(x => string.Format("'Action_{0}'", x));
//if (bUseClFilter)
//{
// DependencyNames.Add(string.Format("'Action_{0}_dep'", ActionIndex));
//}
AddText(string.Format("\t.PreBuildDependencies = {{ {0} }}\n", string.Join(",", DependencyNames.ToArray())));
}
//else if (bUseClFilter)
//{
// AddText(string.Format("\t.PreBuildDependencies = {{ 'Action_{0}_dep' }}\n", ActionIndex));
//}
AddText(string.Format("}}\n\n"));
if (bUseClFilter)
{
string DefResponseFile = DependencyFile + ".response";
using (StreamWriter file = File.CreateText(DefResponseFile))
{
file.Write(string.Format("{0}", string.Join("\n", OtherCompilerOptions), InputFile));
DepsResponseFiles.Add(DefResponseFile);
}
AddText(string.Format("ObjectList('Action_{0}_dep')\n{{\n", ActionIndex));
AddText("\t.Compiler = 'UE4DependencyCompiler'\n");
AddText(string.Format("\t.CompilerOptions = '--nostdout \"%2\" -- {0} @\"%2.response\" \"%1\" /E'\n", DependencySubExec)); // pass /E option to just do preprocess pass only
AddText(string.Format("\t.CompilerOutputPath = '{0}'\n", Path.GetDirectoryName(DependencyFile)));
AddText(string.Format("\t.CompilerOutputExtension = '{0}.txt'\n", Path.GetExtension(InputFile)));
AddText(string.Format("\t.CompilerInputFiles = '{0}'\n", InputFile));
AddText(string.Format("\t.PreBuildDependencies = {{ 'Action_{0}' }}\n", ActionIndex));
//AddText("\t.CompilerInputAllowNoFiles = true\n");
//AddText("\t.AllowDistribution = false\n");
AddText("}\n\n");
}
}
private void AddLinkAction(List<Action> Actions, int ActionIndex, List<int> DependencyIndices)
{
Action Action = Actions[ActionIndex];
string[] SpecialLinkerOptions = { "/OUT:", "@", "-o" };
var ParsedLinkerOptions = ParseCommandLineOptions(Action.CommandArguments, SpecialLinkerOptions, SaveResponseFile: true, SkipInputFile: Action.CommandPath.FullName.Contains("orbis-clang"));
string OutputFile;
if (IsXBOnePDBUtil(Action))
{
OutputFile = ParsedLinkerOptions["OtherOptions"].Trim(' ').Trim('"');
}
else if (IsMSVC())
{
OutputFile = GetOptionValue(ParsedLinkerOptions, "/OUT:", Action, ProblemIfNotFound: true);
}
else //PS4
{
OutputFile = GetOptionValue(ParsedLinkerOptions, "-o", Action, ProblemIfNotFound: false);
if (string.IsNullOrEmpty(OutputFile))
{
OutputFile = GetOptionValue(ParsedLinkerOptions, "InputFile", Action, ProblemIfNotFound: true);
}
}
if (string.IsNullOrEmpty(OutputFile))
{
Console.WriteLine("Failed to find output file. Bailing.");
return;
}
string ResponseFilePath = GetOptionValue(ParsedLinkerOptions, "@", Action);
string OtherCompilerOptions = GetOptionValue(ParsedLinkerOptions, "OtherOptions", Action);
List<int> PrebuildDependencies = new List<int>();
if (IsXBOnePDBUtil(Action))
{
AddText(string.Format("Exec('Action_{0}')\n{{\n", ActionIndex));
AddText(string.Format("\t.ExecExecutable = '{0}'\n", Action.CommandPath));
AddText(string.Format("\t.ExecArguments = '{0}'\n", Action.CommandArguments));
AddText(string.Format("\t.ExecInput = {{ {0} }} \n", ParsedLinkerOptions["InputFile"]));
AddText(string.Format("\t.ExecOutput = '{0}' \n", OutputFile));
AddText(string.Format("\t.PreBuildDependencies = {{ {0} }} \n", ParsedLinkerOptions["InputFile"]));
AddText(string.Format("}}\n\n"));
}
else if (IsPS4SymbolTool(Action))
{
string searchString = "-map=\"";
int execArgumentStart = Action.CommandArguments.LastIndexOf(searchString) + searchString.Length;
int execArgumentEnd = Action.CommandArguments.IndexOf("\"", execArgumentStart);
string ExecOutput = Action.CommandArguments.Substring(execArgumentStart, execArgumentEnd - execArgumentStart);
AddText(string.Format("Exec('Action_{0}')\n{{\n", ActionIndex));
AddText(string.Format("\t.ExecExecutable = '{0}'\n", Action.CommandPath));
AddText(string.Format("\t.ExecArguments = '{0}'\n", Action.CommandArguments));
AddText(string.Format("\t.ExecOutput = '{0}'\n", ExecOutput));
AddText(string.Format("\t.PreBuildDependencies = {{ 'Action_{0}' }} \n", ActionIndex - 1));
AddText(string.Format("}}\n\n"));
}
else if (Action.CommandPath.GetFileName() == "lib.exe" || Action.CommandPath.GetFileName().Contains("orbis-snarl"))
{
if (DependencyIndices.Count > 0)
{
for (int i = 0; i < DependencyIndices.Count; ++i) //Don't specify pch or resource files, they have the wrong name and the response file will have them anyways.
{
int depIndex = DependencyIndices[i];
foreach (FileItem item in Actions[depIndex].ProducedItems)
{
if (item.ToString().Contains(".pch") || item.ToString().Contains(".res"))
{
DependencyIndices.RemoveAt(i);
i--;
PrebuildDependencies.Add(depIndex);
break;
}
}
}
}
AddText(string.Format("Library('Action_{0}')\n{{\n", ActionIndex));
AddText(string.Format("\t.Compiler = '{0}'\n", GetCompilerName()));
if (IsMSVC())
AddText(string.Format("\t.CompilerOptions = '\"%1\" /Fo\"%2\" /c'\n"));
else
AddText(string.Format("\t.CompilerOptions = '\"%1\" -o \"%2\" -c'\n"));
AddText(string.Format("\t.CompilerOutputPath = \"{0}\"\n", Path.GetDirectoryName(OutputFile)));
AddText(string.Format("\t.Librarian = '{0}' \n", Action.CommandPath));
if (!string.IsNullOrEmpty(ResponseFilePath))
{
if (IsMSVC())
// /ignore:4042 to turn off the linker warning about the output option being present twice (command-line + rsp file)
AddText(string.Format("\t.LibrarianOptions = ' /OUT:\"%2\" /ignore:4042 @\"{0}\" \"%1\"' \n", ResponseFilePath));
else if (IsPS4())
AddText(string.Format("\t.LibrarianOptions = '\"%2\" @\"%1\"' \n", ResponseFilePath));
else
AddText(string.Format("\t.LibrarianOptions = '\"%2\" @\"%1\" {0}' \n", OtherCompilerOptions));
}
else
{
if (IsMSVC())
AddText(string.Format("\t.LibrarianOptions = ' /OUT:\"%2\" {0} \"%1\"' \n", OtherCompilerOptions));
}
if (DependencyIndices.Count > 0)
{
List<string> DependencyNames = DependencyIndices.ConvertAll(x => string.Format("'Action_{0}'", x));
if (IsPS4())
AddText(string.Format("\t.LibrarianAdditionalInputs = {{ '{0}' }} \n", ResponseFilePath)); // Hack...Because FastBuild needs at least one Input file
else if (!string.IsNullOrEmpty(ResponseFilePath))
AddText(string.Format("\t.LibrarianAdditionalInputs = {{ {0} }} \n", DependencyNames[0])); // Hack...Because FastBuild needs at least one Input file
else if (IsMSVC())
AddText(string.Format("\t.LibrarianAdditionalInputs = {{ {0} }} \n", string.Join(",", DependencyNames.ToArray())));
PrebuildDependencies.AddRange(DependencyIndices);
}
else
{
string InputFile = GetOptionValue(ParsedLinkerOptions, "InputFile", Action, ProblemIfNotFound: true);
if (InputFile != null && InputFile.Length > 0)
AddText(string.Format("\t.LibrarianAdditionalInputs = {{ '{0}' }} \n", InputFile));
}
if (PrebuildDependencies.Count > 0)
{
List<string> PrebuildDependencyNames = PrebuildDependencies.ConvertAll(x => string.Format("'Action_{0}'", x));
AddText(string.Format("\t.PreBuildDependencies = {{ {0} }} \n", string.Join(",", PrebuildDependencyNames.ToArray())));
}
AddText(string.Format("\t.LibrarianOutput = '{0}' \n", OutputFile));
AddText(string.Format("}}\n\n"));
}
else if (Action.CommandPath.GetFileName() == "link.exe" || Action.CommandPath.GetFileName().Contains("orbis-clang") || Action.CommandPath.GetFileName().Contains("clang++"))
{
if (DependencyIndices.Count > 0) //Insert a dummy node to make sure all of the dependencies are finished.
//If FASTBuild supports PreBuildDependencies on the Executable action we can remove this.
{
string dummyText = string.IsNullOrEmpty(ResponseFilePath) ? GetOptionValue(ParsedLinkerOptions, "InputFile", Action) : ResponseFilePath;
File.SetLastAccessTimeUtc(dummyText, DateTime.UtcNow);
AddText(string.Format("Copy('Action_{0}_dummy')\n{{ \n", ActionIndex));
AddText(string.Format("\t.Source = '{0}' \n", dummyText));
AddText(string.Format("\t.Dest = '{0}' \n", dummyText + ".dummy"));
List<string> DependencyNames = DependencyIndices.ConvertAll(x => string.Format("\t\t'Action_{0}', ;{1}", x, Actions[x].StatusDescription));
AddText(string.Format("\t.PreBuildDependencies = {{\n{0}\n\t}} \n", string.Join("\n", DependencyNames.ToArray())));
AddText(string.Format("}}\n\n"));
}
AddText(string.Format("Executable('Action_{0}')\n{{ \n", ActionIndex));
AddText(string.Format("\t.Linker = '{0}' \n", Action.CommandPath));
if (DependencyIndices.Count == 0)
{
AddText(string.Format("\t.Libraries = {{ '{0}' }} \n", ResponseFilePath));
if (IsMSVC())
{
if (BuildType == FBBuildType.XBOne)
{
AddText(string.Format("\t.LinkerOptions = '/TLBOUT:\"%1\" /Out:\"%2\" @\"{0}\" {1} ' \n", ResponseFilePath, OtherCompilerOptions)); // The TLBOUT is a huge bodge to consume the %1.
}
else
{
// /ignore:4042 to turn off the linker warning about the output option being present twice (command-line + rsp file)
AddText(string.Format("\t.LinkerOptions = '/TLBOUT:\"%1\" /ignore:4042 /Out:\"%2\" @\"{0}\" ' \n", ResponseFilePath)); // The TLBOUT is a huge bodge to consume the %1.
}
}
else
{
AddText(string.Format("\t.LinkerOptions = '{0} -working-directory=\"{1}\" -o \"%2\" @\"%1\"' \n", OtherCompilerOptions, Action.WorkingDirectory)); // The MQ is a huge bodge to consume the %1.
}
}
else
{
AddText(string.Format("\t.Libraries = 'Action_{0}_dummy' \n", ActionIndex));
if (IsMSVC())
{
if (BuildType == FBBuildType.XBOne)
{
AddText(string.Format("\t.LinkerOptions = '/TLBOUT:\"%1\" /Out:\"%2\" @\"{0}\" {1} ' \n", ResponseFilePath, OtherCompilerOptions)); // The TLBOUT is a huge bodge to consume the %1.
}
else
{
AddText(string.Format("\t.LinkerOptions = '/TLBOUT:\"%1\" /Out:\"%2\" @\"{0}\" ' \n", ResponseFilePath)); // The TLBOUT is a huge bodge to consume the %1.
}
}
else
{
AddText(string.Format("\t.LinkerOptions = '{0} -working-directory=\"{1}\" -o \"%2\" @\"%1\"' \n", OtherCompilerOptions, Action.WorkingDirectory)); // The MQ is a huge bodge to consume the %1.
}
}
AddText(string.Format("\t.LinkerOutput = '{0}' \n", OutputFile));
AddText(string.Format("}}\n\n"));
}
else if (Action.CommandPath.GetFileName() == "cmd.exe")
{
AddText(string.Format("Exec('Action_{0}')\n{{\n", ActionIndex));
AddText(string.Format("\t.ExecExecutable = 'C:\\Windows\\System32\\{0}'\n", Action.CommandPath));
AddText(string.Format("\t.ExecArguments = '{0}'\n", Action.CommandArguments));
AddText(string.Format("\t.ExecOutput = '{0}'\n", Action.ProducedItems[0]));
//AddText(string.Format("\t.ExecInput = {{ {0} }} \n", ParsedLinkerOptions["InputFile"]));
//AddText(string.Format("\t.ExecOutput = '{0}' \n", OutputFile));
//AddText(string.Format("\t.PreBuildDependencies = {{ {0} }} \n", ParsedLinkerOptions["InputFile"]));
AddText(string.Format("}}\n\n"));
}
}
private FileStream bffOutputFileStream = null;
private bool CreateBffFile(List<Action> InActions, string BffFilePath, List<Action> LocalExecutorActions, out int NumFastbuildActions)
{
List<Action> Actions = SortActions(InActions);
try
{
bffOutputFileStream = new FileStream(BffFilePath, FileMode.Create, FileAccess.Write);
WriteEnvironmentSetup(Actions); //Compiler, environment variables and base paths
List<int> FastBuildActionIndices = new List<int>();
for (int ActionIndex = 0; ActionIndex < Actions.Count; ActionIndex++)
{
Action Action = Actions[ActionIndex];
// Resolve dependencies
List<int> DependencyIndices = new List<int>();
foreach (Action RequiredAction in Action.PrerequisiteActions)
{
int ProducingActionIndex = Actions.IndexOf(RequiredAction);
if (ProducingActionIndex >= 0)
{
DependencyIndices.Add(ProducingActionIndex);
}
}
AddText(string.Format("// \"{0}\" {1}\n", Action.CommandPath, Action.CommandArguments));
switch (Action.ActionType)
{
case ActionType.Compile: AddCompileAction(Action, ActionIndex, DependencyIndices); FastBuildActionIndices.Add(ActionIndex); break;
case ActionType.Link: AddLinkAction(Actions, ActionIndex, DependencyIndices); FastBuildActionIndices.Add(ActionIndex); break;
case ActionType.WriteMetadata: LocalExecutorActions.Add(Action); break;
case ActionType.BuildProject: LocalExecutorActions.Add(Action); break;
case ActionType.PostBuildStep: LocalExecutorActions.Add(Action); break;
default: Console.WriteLine("Fastbuild is ignoring an unsupported action: " + Action.ActionType.ToString()); break;
}
}
AddText("Alias( 'all' ) \n{\n");
AddText("\t.Targets = { \n");
foreach (int ActionIndex in FastBuildActionIndices)
{
bool bUseClFilter = Actions[ActionIndex].CommandPath.GetFileName() == "cl-filter.exe";
if (bUseClFilter)
{
AddText(string.Format("\t\t'Action_{0}_dep'{1}", ActionIndex, ActionIndex < FastBuildActionIndices.Last() ? ",\n" : ""));
}
else
{
AddText(string.Format("\t\t'Action_{0}'{1}", ActionIndex, ActionIndex < FastBuildActionIndices.Last() ? ",\n" : ""));
}
}
AddText("\t}\n}\n");
bffOutputFileStream.Close();
NumFastbuildActions = FastBuildActionIndices.Count;
}
catch (Exception e)
{
Console.WriteLine("Exception while creating bff file: " + e.ToString());
NumFastbuildActions = 0;
return false;
}
return true;
}
private bool ExecuteBffFile(string BffFilePath)
{
string cacheArgument = "";
if (bEnableCaching)
{
switch (CacheMode)
{
case eCacheMode.ReadOnly:
cacheArgument = "-cacheread";
break;
case eCacheMode.WriteOnly:
cacheArgument = "-cachewrite";
break;
case eCacheMode.ReadWrite:
cacheArgument = "-cache";
break;
}
}
string distArgument = bEnableDistribution ? "-dist" : "";
//Interesting flags for FASTBuild: -nostoponerror, -verbose, -monitor (if FASTBuild Monitor Visual Studio Extension is installed!)
// Yassine: The -clean is to bypass the FastBuild internal dependencies checks (cached in the fdb) as it could create some conflicts with UBT.
// Basically we want FB to stupidly compile what UBT tells it to.
string FBCommandLine = string.Format("-monitor -summary {0} {1} -ide -clean -distverbose -nostoponerror -config \"{2}\"", distArgument, cacheArgument, BffFilePath);
Console.WriteLine("Executing " + FBCommandLine);
ProcessStartInfo FBStartInfo = new ProcessStartInfo(string.IsNullOrEmpty(FBuildExePathOverride) ? "fbuild" : FBuildExePathOverride, FBCommandLine);
FBStartInfo.UseShellExecute = false;
FBStartInfo.WorkingDirectory = Path.Combine(UnrealBuildTool.EngineDirectory.MakeRelativeTo(DirectoryReference.GetCurrentDirectory()), "Source");
try
{
Process FBProcess = new Process();
FBProcess.StartInfo = FBStartInfo;
FBStartInfo.RedirectStandardError = true;
FBStartInfo.RedirectStandardOutput = true;
FBProcess.EnableRaisingEvents = true;
DataReceivedEventHandler OutputEventHandler = (Sender, Args) =>
{
if (Args.Data != null)
Console.WriteLine(Args.Data);
};
FBProcess.OutputDataReceived += OutputEventHandler;
FBProcess.ErrorDataReceived += OutputEventHandler;
FBProcess.Start();
FBProcess.BeginOutputReadLine();
FBProcess.BeginErrorReadLine();
FBProcess.WaitForExit();
return FBProcess.ExitCode == 0;
}
catch (Exception e)
{
Console.WriteLine("Exception launching fbuild process. Is it in your path?" + e.ToString());
return false;
}
finally
{
//foreach (string DepResponseFile in DepsResponseFiles)
//{
// File.Delete(DepResponseFile);
//}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment