Add Incremental builds for exes with no output (#117)

* Add support for executable and evaluate timestamp incremental builds
* Bump version

Source/Client/CLI/Recipe.toml

@@ -1,6 +1,6 @@
 Name = "Soup"
 Language = "C++"
-Version = "0.17.2"
+Version = "0.17.3"
 Type = "Executable"
 
 Source = [

Source/Client/CLI/Source/Commands/VersionCommand.h

@@ -31,7 +31,7 @@ namespace Soup::Client
 
 			// TODO var version = Assembly.GetExecutingAssembly().GetName().Version;
 			// Log::Message($"{version.Major}.{version.Minor}.{version.Build}");
-			Log::HighPriority("0.17.2");
+			Log::HighPriority("0.17.3");
 		}
 
 	private:

Source/Client/Core/Source/Build/BuildEvaluateEngine.h

@@ -106,10 +106,21 @@ namespace Soup::Core
 			auto buildRequired = false;
 			if (operationInfo.WasSuccessfulRun)
 			{
+				// Check if the executable has changed since the last run
+				bool executableOutOfDate = false;
+				if (operationInfo.Command.Executable != Path("writefile.exe"))
+				{
+					// Only check for "real" executables
+					auto executableFileId = _fileSystemState->ToFileId(operationInfo.Command.Executable, operationInfo.Command.WorkingDirectory);
+					if (_stateChecker.IsOutdated(operationInfo.EvaluateTime, executableFileId))
+					{
+						executableOutOfDate = true;
+					}
+				}
+
 				// Perform the incremental build checks
-				if (_stateChecker.IsOutdated(
-					operationInfo.ObservedOutput,
-					operationInfo.ObservedInput))
+				if (executableOutOfDate ||
+					_stateChecker.IsOutdated(operationInfo.ObservedOutput, operationInfo.ObservedInput))
 				{
 					buildRequired = true;
 				}
@@ -185,6 +196,7 @@ namespace Soup::Core
 
 			// Mark this operation as successful to enable future incremental builds
 			operationInfo.WasSuccessfulRun = true;
+			operationInfo.EvaluateTime = std::chrono::system_clock::now();
 
 			// Ensure the File System State is notified of any output files that have changed
 			_fileSystemState->CheckFileWriteTimes(operationInfo.ObservedOutput);
@@ -282,11 +294,9 @@ namespace Soup::Core
 				operationInfo.ObservedInput = _fileSystemState->ToFileIds(input, operationInfo.Command.WorkingDirectory);
 				operationInfo.ObservedOutput = _fileSystemState->ToFileIds(output, operationInfo.Command.WorkingDirectory);
 
-				// Add the executable as actual input too
-				// TODO: operationInfo.ObservedInput.push_back(_fileSystemState->ToFileId(operationInfo.Command.Executable, operationInfo.Command.WorkingDirectory));
-
 				// Mark this operation as successful to enable future incremental builds
 				operationInfo.WasSuccessfulRun = true;
+				operationInfo.EvaluateTime = std::chrono::system_clock::now();
 
 				// Ensure the File System State is notified of any output files that have changed
 				_fileSystemState->CheckFileWriteTimes(operationInfo.ObservedOutput);

Source/Client/Core/Source/Build/BuildHistoryChecker.h

@@ -15,6 +15,39 @@ namespace Soup::Core
 		{
 		}
 
+		/// <summary>
+		/// Perform a check if the last evaluate time is outdated with
+		/// respect to the input files
+		/// </summary>
+		bool IsOutdated(
+			std::chrono::time_point<std::chrono::system_clock> lastEvaluateTime,
+			FileId inputFile)
+		{
+			auto lastWriteTime = _fileSystemState->GetLastWriteTime(inputFile);
+
+			// Perform the final check
+			if (!lastWriteTime.has_value())
+			{
+				// The input was missing
+				auto targetFilePath = _fileSystemState->GetFilePath(inputFile);
+				Log::Info("Input Missing [" + targetFilePath.ToString() + "]");
+				return true;
+			}
+			else
+			{
+				if (lastWriteTime.value() > lastEvaluateTime)
+				{
+					auto targetFilePath = _fileSystemState->GetFilePath(inputFile);
+					Log::Info("Input altered after last evaluate [" + targetFilePath.ToString() + "]");
+					return true;
+				}
+				else
+				{
+					return false;
+				}
+			}
+		}
+
 		/// <summary>
 		/// Perform a check if the requested target is outdated with
 		/// respect to the input files
@@ -48,7 +81,7 @@ namespace Soup::Core
 			const std::vector<FileId>& inputFiles)
 		{
 			// Get the output file last write time
-			std::optional<time_t> targetFileLastWriteTime = _fileSystemState->GetLastWriteTime(targetFile);
+			auto targetFileLastWriteTime = _fileSystemState->GetLastWriteTime(targetFile);
 
 			if (!targetFileLastWriteTime.has_value())
 			{
@@ -70,10 +103,13 @@ namespace Soup::Core
 			return false;
 		}
 
-		bool IsOutdated(FileId inputFile, FileId outputFile, std::time_t outputFileLastWriteTime)
+		bool IsOutdated(
+			FileId inputFile,
+			FileId outputFile,
+			std::chrono::time_point<std::chrono::system_clock> outputFileLastWriteTime)
 		{
 			// Get the file state from the cache
-			std::optional<time_t> lastWriteTime = _fileSystemState->GetLastWriteTime(inputFile);
+			auto lastWriteTime = _fileSystemState->GetLastWriteTime(inputFile);
 
 			// Perform the final check
 			if (!lastWriteTime.has_value())

Source/Client/Core/Source/Build/FileSystemState.h

@@ -41,7 +41,7 @@ namespace Soup::Core
 		FileSystemState(
 			FileId maxFileId,
 			std::unordered_map<FileId, Path> files,
-			std::unordered_map<FileId, std::optional<time_t>> writeCache) :
+			std::unordered_map<FileId, std::optional<std::chrono::time_point<std::chrono::system_clock>>> writeCache) :
 			_maxFileId(maxFileId),
 			_files(std::move(files)),
 			_fileLookup(),
@@ -86,7 +86,7 @@ namespace Soup::Core
 		/// <summary>
 		/// Find the write time for a given file id
 		/// </summary>
-		std::optional<time_t> GetLastWriteTime(FileId file)
+		std::optional<std::chrono::time_point<std::chrono::system_clock>> GetLastWriteTime(FileId file)
 		{
 			auto findResult = _writeCache.find(file);
 			if (findResult != _writeCache.end())
@@ -196,16 +196,17 @@ namespace Soup::Core
 		/// <summary>
 		/// Update the write times for the provided set of files
 		/// </summary>
-		std::optional<time_t> CheckFileWriteTime(FileId fileId)
+		std::optional<std::chrono::time_point<std::chrono::system_clock>> CheckFileWriteTime(FileId fileId)
 		{
 			auto& filePath = GetFilePath(fileId);
 
 			// The file does not exist in the cache
 			// Load the actual value and save it for later
-			std::optional<std::time_t> lastWriteTime = std::nullopt;
+			std::optional<std::chrono::time_point<std::chrono::system_clock>> lastWriteTime = std::nullopt;
 			if (System::IFileSystem::Current().Exists(filePath))
 			{
-				lastWriteTime = System::IFileSystem::Current().GetLastWriteTime(filePath);
+				lastWriteTime = std::chrono::system_clock::from_time_t(
+					System::IFileSystem::Current().GetLastWriteTime(filePath));
 			}
 
 			auto insertResult = _writeCache.insert_or_assign(fileId, lastWriteTime);
@@ -220,6 +221,6 @@ namespace Soup::Core
 		std::unordered_map<FileId, Path> _files;
 		std::unordered_map<std::string, FileId> _fileLookup;
 
-		std::unordered_map<FileId, std::optional<time_t>> _writeCache;
+		std::unordered_map<FileId, std::optional<std::chrono::time_point<std::chrono::system_clock>>> _writeCache;
 	};
 }

Source/Client/Core/Source/Build/RecipeBuildRunner.h

@@ -727,6 +727,7 @@ namespace Soup::Core
 					if (previousOperationGraph.TryFindOperationInfo(activeOperationInfo.Command, previousOperationInfo))
 					{
 						activeOperationInfo.WasSuccessfulRun = previousOperationInfo->WasSuccessfulRun;
+						activeOperationInfo.EvaluateTime = previousOperationInfo->EvaluateTime;
 						activeOperationInfo.ObservedInput = previousOperationInfo->ObservedInput;
 						activeOperationInfo.ObservedOutput = previousOperationInfo->ObservedOutput;
 					}

Source/Client/Core/Source/Module.cpp

@@ -24,12 +24,9 @@
 
 #include <Windows.h>
 
-#ifdef max
 #undef max
-#endif
-#ifdef CreateProcess
+#undef min
 #undef CreateProcess
-#endif
 
 #include <openssl/evp.h>
 

Source/Client/Core/Source/OperationGraph/OperationGraphReader.h

@@ -14,7 +14,10 @@ namespace Soup::Core
 	{
 	private:
 		// Binary Operation Graph file format
-		static constexpr uint32_t FileVersion = 3;
+		static constexpr uint32_t FileVersion = 4;
+
+		// The offset from January 1, 1970 at 00:00:00.000 to January 1, 0001 at 00:00:00.000 in the Gregorian calendar
+		static constexpr long long UnixEpochOffset = 62135596800000;
 
 	public:
 		static OperationGraph Deserialize(std::istream& stream)
@@ -139,6 +142,11 @@ namespace Soup::Core
 			// Write out the value indicating if there was a successful run
 			auto wasSuccessfulRun = ReadBoolean(stream);
 
+			// Read the utc tick since January 1, 0001 at 00:00:00.000 in the Gregorian calendar
+			auto evaluateTimeMilliseconds = ReadInt64(stream);
+			auto unixEvaluateTimeMilliseconds = std::chrono::milliseconds(evaluateTimeMilliseconds - UnixEpochOffset);
+			auto evaluateTime = std::chrono::time_point<std::chrono::system_clock>(unixEvaluateTimeMilliseconds);
+
 			// Write out the observed input files
 			auto observedInput = ReadFileIdList(stream);
 
@@ -159,6 +167,7 @@ namespace Soup::Core
 				std::move(children),
 				dependecyCount,
 				wasSuccessfulRun,
+				evaluateTime,
 				std::move(observedInput),
 				std::move(observedOutput));
 		}
@@ -175,6 +184,18 @@ namespace Soup::Core
 			return result;
 		}
 
+		static int64_t ReadInt64(std::istream& stream)
+		{
+			int64_t result = 0;
+			stream.read(reinterpret_cast<char*>(&result), sizeof(int64_t));
+			if (stream.fail())
+			{
+				throw std::runtime_error("OperationGraphReader Failed to read 64 bit integer value");
+			}
+
+			return result;
+		}
+
 		static boolean ReadBoolean(std::istream& stream)
 		{
 			uint32_t result = 0;

Source/Client/Core/Source/OperationGraph/OperationGraphWriter.h

@@ -14,7 +14,10 @@ namespace Soup::Core
 	{
 	private:
 		// Binary Operation graph file format
-		static constexpr uint32_t FileVersion = 3;
+		static constexpr uint32_t FileVersion = 4;
+
+		// The offset from January 1, 1970 at 00:00:00.000 to January 1, 0001 at 00:00:00.000 in the Gregorian calendar
+		static constexpr long long UnixEpochOffset = 62135596800000;
 
 	public:
 		static void Serialize(const OperationGraph& state, std::ostream& stream)
@@ -87,6 +90,11 @@ namespace Soup::Core
 			// Write out the value indicating if there was a successful run
 			WriteValue(stream, operation.WasSuccessfulRun);
 
+			// Write out the utc milliseconds since January 1, 0001 at 00:00:00.000 in the Gregorian calendar
+			auto unixEvaluateTimeMilliseconds = std::chrono::time_point_cast<std::chrono::milliseconds>(operation.EvaluateTime).time_since_epoch().count();
+			auto evaluateTimeMilliseconds = unixEvaluateTimeMilliseconds + UnixEpochOffset;
+			WriteValue(stream, evaluateTimeMilliseconds);
+
 			// Write out the observed input files
 			WriteValues(stream, operation.ObservedInput);
 
@@ -99,6 +107,11 @@ namespace Soup::Core
 			stream.write(reinterpret_cast<char*>(&value), sizeof(uint32_t));
 		}
 
+		static void WriteValue(std::ostream& stream, int64_t value)
+		{
+			stream.write(reinterpret_cast<char*>(&value), sizeof(int64_t));
+		}
+
 		static void WriteValue(std::ostream& stream, bool value)
 		{
 			uint32_t integerValue = value ? 1u : 0u;

Source/Client/Core/Source/OperationGraph/OperationInfo.h

@@ -4,6 +4,7 @@
 
 #pragma once
 #include "FileSystemState.h"
+using namespace std::chrono_literals;
 
 namespace Soup::Core
 {
@@ -67,6 +68,7 @@ namespace Soup::Core
 			Children(),
 			DependencyCount(0),
 			WasSuccessfulRun(false),
+			EvaluateTime(std::chrono::time_point<std::chrono::system_clock>::min()),
 			ObservedInput(),
 			ObservedOutput()
 		{
@@ -93,6 +95,7 @@ namespace Soup::Core
 			Children(),
 			DependencyCount(0),
 			WasSuccessfulRun(false),
+			EvaluateTime(std::chrono::time_point<std::chrono::system_clock>::min()),
 			ObservedInput(),
 			ObservedOutput()
 		{
@@ -112,6 +115,7 @@ namespace Soup::Core
 			std::vector<OperationId> children,
 			uint32_t dependencyCount,
 			bool wasSuccessfulRun,
+			std::chrono::time_point<std::chrono::system_clock> evaluateTime,
 			std::vector<FileId> observedInput,
 			std::vector<FileId> observedOutput) :
 			Id(id),
@@ -124,6 +128,7 @@ namespace Soup::Core
 			Children(std::move(children)),
 			DependencyCount(dependencyCount),
 			WasSuccessfulRun(wasSuccessfulRun),
+			EvaluateTime(evaluateTime),
 			ObservedInput(std::move(observedInput)),
 			ObservedOutput(std::move(observedOutput))
 		{
@@ -144,6 +149,7 @@ namespace Soup::Core
 				Children == rhs.Children &&
 				DependencyCount == rhs.DependencyCount &&
 				WasSuccessfulRun == rhs.WasSuccessfulRun &&
+				EvaluateTime == rhs.EvaluateTime &&
 				ObservedInput == rhs.ObservedInput &&
 				ObservedOutput == rhs.ObservedOutput;
 		}
@@ -158,6 +164,7 @@ namespace Soup::Core
 		std::vector<OperationId> Children;
 		uint32_t DependencyCount;
 		bool WasSuccessfulRun;
+		std::chrono::time_point<std::chrono::system_clock> EvaluateTime;
 		std::vector<FileId> ObservedInput;
 		std::vector<FileId> ObservedOutput;
 	};