-
Notifications
You must be signed in to change notification settings - Fork 270
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Retry platform-level errors in the isolated process for .NET isolated (…
…#2922) Co-authored-by: Andy Staples <[email protected]>
- Loading branch information
1 parent
1163944
commit 8a5db71
Showing
7 changed files
with
447 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
test/SmokeTests/OOProcSmokeTests/DotNetIsolated/DotNetIsolated.sln
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
|
|
||
| Microsoft Visual Studio Solution File, Format Version 12.00 | ||
| # Visual Studio Version 17 | ||
| VisualStudioVersion = 17.5.002.0 | ||
| MinimumVisualStudioVersion = 10.0.40219.1 | ||
| Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetIsolated", "DotNetIsolated.csproj", "{B2DBA49D-9D25-46DB-8968-15D5E83B4060}" | ||
| EndProject | ||
| Global | ||
| GlobalSection(SolutionConfigurationPlatforms) = preSolution | ||
| Debug|Any CPU = Debug|Any CPU | ||
| Release|Any CPU = Release|Any CPU | ||
| EndGlobalSection | ||
| GlobalSection(ProjectConfigurationPlatforms) = postSolution | ||
| {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {B2DBA49D-9D25-46DB-8968-15D5E83B4060}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| EndGlobalSection | ||
| GlobalSection(SolutionProperties) = preSolution | ||
| HideSolutionNode = FALSE | ||
| EndGlobalSection | ||
| GlobalSection(ExtensibilityGlobals) = postSolution | ||
| SolutionGuid = {0954D7B4-582F-4F85-AE3E-5D503FB07DB1} | ||
| EndGlobalSection | ||
| EndGlobal |
165 changes: 165 additions & 0 deletions
165
test/SmokeTests/OOProcSmokeTests/DotNetIsolated/FaultyOrchestrators.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| using Microsoft.Azure.Functions.Worker; | ||
| using Microsoft.Azure.Functions.Worker.Http; | ||
| using Microsoft.DurableTask; | ||
| using Microsoft.DurableTask.Client; | ||
| using Microsoft.Extensions.Logging; | ||
| using System; | ||
|
|
||
| namespace FaultOrchestrators | ||
| { | ||
| public static class FaultyOrchestrators | ||
| { | ||
| [Function(nameof(OOMOrchestrator))] | ||
| public static Task OOMOrchestrator( | ||
| [OrchestrationTrigger] TaskOrchestrationContext context) | ||
| { | ||
| // this orchestrator is not deterministic, on purpose. | ||
| // we use the non-determinism to force an OOM exception on only the first replay | ||
|
|
||
| // check if a file named "replayEvidence" exists in source code directory, create it if it does not. | ||
| // From experience, this code runs in `<sourceCodePath>/bin/output/`, so we store the file two directories above. | ||
| // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically | ||
| // when `func host start` is re-invoked. | ||
| string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); | ||
| bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); | ||
| if (isTheFirstReplay) | ||
| { | ||
| System.IO.File.Create(evidenceFile).Close(); | ||
|
|
||
| // force the process to run out of memory | ||
| List<byte[]> data = new List<byte[]>(); | ||
|
|
||
| for (int i = 0; i < 10000000; i++) | ||
| { | ||
| data.Add(new byte[1024 * 1024 * 1024]); | ||
| } | ||
|
|
||
| // we expect the code to never reach this statement, it should OOM. | ||
| // we throw just in case the code does not time out. This should fail the test | ||
| throw new Exception("this should never be reached"); | ||
| } | ||
| else { | ||
| // if it's not the first replay, delete the evidence file and return | ||
| System.IO.File.Delete(evidenceFile); | ||
| return Task.CompletedTask; | ||
| } | ||
| } | ||
|
|
||
| [Function(nameof(ProcessExitOrchestrator))] | ||
| public static Task ProcessExitOrchestrator( | ||
| [OrchestrationTrigger] TaskOrchestrationContext context) | ||
| { | ||
| // this orchestrator is not deterministic, on purpose. | ||
| // we use the non-determinism to force a sudden process exit on only the first replay | ||
|
|
||
| // check if a file named "replayEvidence" exists in source code directory, create it if it does not. | ||
| // From experience, this code runs in `<sourceCodePath>/bin/output/`, so we store the file two directories above. | ||
| // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically | ||
| // when `func host start` is re-invoked. | ||
| string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); | ||
| bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); | ||
| if (isTheFirstReplay) | ||
| { | ||
| System.IO.File.Create(evidenceFile).Close(); | ||
|
|
||
| // force sudden crash | ||
| Environment.FailFast("Simulating crash!"); | ||
| throw new Exception("this should never be reached"); | ||
| } | ||
| else { | ||
| // if it's not the first replay, delete the evidence file and return | ||
| System.IO.File.Delete(evidenceFile); | ||
| return Task.CompletedTask; | ||
| } | ||
| } | ||
|
|
||
| [Function(nameof(TimeoutOrchestrator))] | ||
| public static Task TimeoutOrchestrator( | ||
| [OrchestrationTrigger] TaskOrchestrationContext context) | ||
| { | ||
| // this orchestrator is not deterministic, on purpose. | ||
| // we use the non-determinism to force a timeout on only the first replay | ||
|
|
||
| // check if a file named "replayEvidence" exists in source code directory, create it if it does not. | ||
| // From experience, this code runs in `<sourceCodePath>/bin/output/`, so we store the file two directories above. | ||
| // We do this because the /bin/output/ directory gets overridden during the build process, which happens automatically | ||
| // when `func host start` is re-invoked. | ||
| string evidenceFile = System.IO.Path.Combine(System.IO.Directory.GetCurrentDirectory(), "..", "..", "replayEvidence"); | ||
| bool isTheFirstReplay = !System.IO.File.Exists(evidenceFile); | ||
|
|
||
| if (isTheFirstReplay) | ||
| { | ||
| System.IO.File.Create(evidenceFile).Close(); | ||
|
|
||
| // force the process to timeout after a 1 minute wait | ||
| System.Threading.Thread.Sleep(TimeSpan.FromMinutes(1)); | ||
|
|
||
| // we expect the code to never reach this statement, it should time out. | ||
| // we throw just in case the code does not time out. This should fail the test | ||
| throw new Exception("this should never be reached"); | ||
| } | ||
| else { | ||
| // if it's not the first replay, delete the evidence file and return | ||
| System.IO.File.Delete(evidenceFile); | ||
| return Task.CompletedTask; | ||
| } | ||
| } | ||
|
|
||
| [Function("durable_HttpStartOOMOrchestrator")] | ||
| public static async Task<HttpResponseData> HttpStartOOMOrchestrator( | ||
| [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, | ||
| [DurableClient] DurableTaskClient client, | ||
| FunctionContext executionContext) | ||
| { | ||
| ILogger logger = executionContext.GetLogger("durable_HttpStartOOMOrchestrator"); | ||
|
|
||
| // Function input comes from the request content. | ||
| string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( | ||
| nameof(OOMOrchestrator)); | ||
|
|
||
| logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); | ||
|
|
||
| // Returns an HTTP 202 response with an instance management payload. | ||
| // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration | ||
| return await client.CreateCheckStatusResponseAsync(req, instanceId); | ||
| } | ||
|
|
||
| [Function("durable_HttpStartProcessExitOrchestrator")] | ||
| public static async Task<HttpResponseData> HttpStartProcessExitOrchestrator( | ||
| [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, | ||
| [DurableClient] DurableTaskClient client, | ||
| FunctionContext executionContext) | ||
| { | ||
| ILogger logger = executionContext.GetLogger("durable_HttpStartProcessExitOrchestrator"); | ||
|
|
||
| // Function input comes from the request content. | ||
| string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( | ||
| nameof(ProcessExitOrchestrator)); | ||
|
|
||
| logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); | ||
|
|
||
| // Returns an HTTP 202 response with an instance management payload. | ||
| // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration | ||
| return await client.CreateCheckStatusResponseAsync(req, instanceId); | ||
| } | ||
|
|
||
| [Function("durable_HttpStartTimeoutOrchestrator")] | ||
| public static async Task<HttpResponseData> HttpStartTimeoutOrchestrator( | ||
| [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, | ||
| [DurableClient] DurableTaskClient client, | ||
| FunctionContext executionContext) | ||
| { | ||
| ILogger logger = executionContext.GetLogger("durable_HttpStartTimeoutOrchestrator"); | ||
|
|
||
| // Function input comes from the request content. | ||
| string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( | ||
| nameof(TimeoutOrchestrator)); | ||
|
|
||
| logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); | ||
|
|
||
| // Returns an HTTP 202 response with an instance management payload. | ||
| // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration | ||
| return await client.CreateCheckStatusResponseAsync(req, instanceId); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.