Moving to container based app on EC2 instead of directly running on the instance

.github/workflows/build.yml

@@ -17,6 +17,7 @@ jobs:
       CDK_LOCATION: "cdk"
       PROJECT_NAME: ${{ github.event.repository.name }}
       DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 0
+      BUILD_APP: false
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -150,6 +151,7 @@ jobs:
         run: curl -SL https://github.com/docker/compose/releases/download/v2.32.4/docker-compose-linux-aarch64 -o ${{ github.workspace }}/assets/docker-compose
 
       - name: Build arm64 web app
+        if: env.BUILD_APP == true
         run: |
           rm -rf ${{ github.workspace }}/app-src/output
           mkdir -p ${{ github.workspace }}/app-src/output
@@ -164,6 +166,7 @@ jobs:
           rm -rf ${{ github.workspace }}/app-src/output
 
       - name: Build failing arm64 web app
+        if: env.BUILD_APP == true
         run: |
           rm -rf ${{ github.workspace }}/app-src/output
           mkdir -p ${{ github.workspace }}/app-src/output

app-src/Program.cs

@@ -1,18 +1,7 @@
-using Amazon.CloudWatch.EMF.Config;
 using BAMCIS.MultiAZApp.Utilities;
 
 var builder = WebApplication.CreateBuilder(args);
 
-EnvironmentConfigurationProvider.Config = new Configuration
-{
-    ServiceName = Constants.SERVICE_NAME,
-    LogGroupName = Constants.LOG_GROUP_NAME,
-    ServiceType =  "WebApi",
-    EnvironmentOverride = builder.Environment.IsDevelopment()
-        ? Amazon.CloudWatch.EMF.Environment.Environments.Local
-        : Amazon.CloudWatch.EMF.Environment.Environments.EC2
-};
-
 builder.WebHost.ConfigureKestrel((context, serverOptions) => {
     serverOptions.AddServerHeader = true;
     serverOptions.ListenAnyIP(5000);

app-src/Utilities/ServiceCollectionExtensions.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics;
+using Amazon.CloudWatch.EMF.Config;
 using Amazon.CloudWatch.EMF.Logger;
 using Amazon.CloudWatch.EMF.Model;
 using Amazon.XRay.Recorder.Core;
@@ -57,14 +58,34 @@ public static WebApplicationBuilder RegisterServices(this WebApplicationBuilder
             // cache refresh worker
             builder.Services.AddHostedService<BackgroundWorker>();
             builder.Services.AddControllers();        
-            builder.Services.AddEmf();
+
+            //builder.Services.AddSerilog();
+            builder.Services.AddEmf(builder.Environment);
             builder.Services.AddOpenApi();
 
             return builder;
         }
 
-        private static void AddEmf(this IServiceCollection services)
+        private static void AddEmf(this IServiceCollection services, IWebHostEnvironment env)
         {
+            Console.WriteLine("IS DEVELOPMENT: " + env.IsDevelopment());
+            Console.WriteLine("ENV: " + env.EnvironmentName);
+            EnvironmentConfigurationProvider.Config = new Configuration(
+                serviceName: Constants.SERVICE_NAME,
+                serviceType: "WebApi",
+                logGroupName: Constants.LOG_GROUP_NAME,
+                logStreamName: String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("AWS_EMF_LOG_STREAM_NAME")) ?
+                    String.Empty :
+                    System.Environment.GetEnvironmentVariable("AWS_EMF_LOG_STREAM_NAME"),
+                agentEndPoint: String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("AWS_EMF_AGENT_ENDPOINT")) ?
+                    Amazon.CloudWatch.EMF.Sink.Endpoint.DEFAULT_TCP_ENDPOINT.ToString() :
+                    System.Environment.GetEnvironmentVariable("AWS_EMF_AGENT_ENDPOINT"),
+                agentBufferSize: Configuration.DEFAULT_AGENT_BUFFER_SIZE,
+                environmentOverride: env != null && env.IsDevelopment() ? 
+                    Amazon.CloudWatch.EMF.Environment.Environments.Local :
+                    Amazon.CloudWatch.EMF.Environment.Environments.Agent
+            );
+
             services.AddScoped<IMetricsLogger, MetricsLogger>();
             services.AddSingleton<Amazon.CloudWatch.EMF.Environment.IEnvironmentProvider, Amazon.CloudWatch.EMF.Environment.EnvironmentProvider>();
             services.AddSingleton<Amazon.CloudWatch.EMF.Environment.IResourceFetcher, Amazon.CloudWatch.EMF.Environment.ResourceFetcher>();

app-src/appsettings.json

@@ -1,7 +1,7 @@
 {
   "Logging": {
     "LogLevel": {
-      "Default": "Information",
+      "Default": "Debug",
       "Microsoft.AspNetCore": "Warning"
     }
   },

app-src/docker/docker-compose.yml

@@ -8,8 +8,11 @@ services:
     ports:
       - 5000:5000
     environment:
-      - "AWS_EMF_AGENT_ENDPOINT=tcp://cwagent:25889"
+      - "AWS_EMF_AGENT_ENDPOINT=tcp://cwagent:25888"
+      - "AWS_EMF_LOG_STREAM_NAME=${AWS_EMF_LOG_STREAM_NAME}"
       - "DB_SECRET=${DB_SECRET}"
+      - "AWS_XRAY_DAEMON_ADDRESS=cwagent:2000"
+      - "OTEL_EXPORTER_OTLP_ENDPOINT=http://cwagent:4318"
   # healthcheck:
   #    test: curl --fail --location http://localhost:5000/health || exit 1
   #    interval: 5s
@@ -22,7 +25,5 @@ services:
     mem_limit: 256M
     cpus: 0.25
     restart: always
-    ports: 
-      - 25889:25889/tcp
     environment:
-      - "CW_CONFIG_CONTENT={\"logs\": {\"metrics_collected\": {\"emf\": { \"service_address\": \"tcp://0.0.0.0:25889\" }}}}"
+      - "CW_CONFIG_CONTENT={\"logs\": {\"metrics_collected\": {\"emf\": { } } }, \"traces\": { \"traces_collected\": {\"xray\": {}, \"otlp\": {} }}}"

app-src/docker/load-container.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-rootDirectory="/opt/codedeploy-agent/deployment-root"
-app="$rootDirectory/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/container.tar.gz"
-cw="$rootDirectory/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/cloudwatch-agent.tar.gz"
+ROOT_DIR="/opt/codedeploy-agent/deployment-root"
+APP="$ROOT_DIR/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/container.tar.gz"
+CW="$ROOT_DIR/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/cloudwatch-agent.tar.gz"
 
-docker load < $app
-docker load < $cw
+/usr/bin/docker load < $APP
+/usr/bin/docker load < $CW

app-src/docker/remove-container.sh

@@ -2,4 +2,4 @@
 
 rootDirectory="/opt/codedeploy-agent/deployment-root"
 compose="$rootDirectory/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/docker/docker-compose.yml"
-docker compose --file $compose rm --force --stop --volumes
+/usr/bin/docker compose --file $compose rm --force --stop --volumes

app-src/docker/start-container.sh

@@ -1,8 +1,11 @@
 #!/bin/bash
 
-rootDirectory="/opt/codedeploy-agent/deployment-root"
-compose="$rootDirectory/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/docker/docker-compose.yml"
-env="$rootDirectory/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/docker/.env"
+ROOT_DIR="/opt/codedeploy-agent/deployment-root"
+COMPOSE_FILE="$ROOT_DIR/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/docker/docker-compose.yml"
+ENV_FILE="$ROOT_DIR/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/docker/.env"
+TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id)
 DB_SECRET=$(cat /etc/secret)
-echo "DB_SECRET=$DB_SECRET" >> $env
-docker compose --file $compose --env-file $env up --detach
+echo "DB_SECRET=$DB_SECRET" >> $ENV_FILE
+echo "AWS_EMF_LOG_STREAM_NAME=$INSTANCE_ID" >> $ENV_FILE
+/usr/bin/docker compose --file $COMPOSE_FILE --env-file $ENV_FILE up --detach

app-src/docker/stop-container.sh

@@ -6,9 +6,9 @@ CONTAINER2="cwagent"
 # Function to stop a container if it exists
 stop_container() {
     local container_name=$1
-    if docker ps -q -f name="^${container_name}$" | grep -q .; then
+    if /usr/bin/docker ps -q -f name="^${container_name}$" | grep -q .; then
         echo "Stopping container: $container_name"
-        docker stop "$container_name"
+        /usr/bin/docker stop "$container_name"
     else
         echo "Container $container_name is not running or does not exist."
     fi

app-src/multi-az-workshop-application.csproj

@@ -21,6 +21,8 @@
     <PackageReference Include="AWSSDK.SecretsManager" Version="3.7.400.59" />
     <PackageReference Include="Npgsql" Version="9.0.2" />
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
+    <PackageReference Include="Serilog.AspNetCore" Version="9.0.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
   </ItemGroup>
 
 </Project>

cdk/Configs/cw-agent-config.json

@@ -45,29 +45,35 @@
                     {
                         "file_path": "/var/log/cfn-init-cmd.log",
                         "log_group_name": "cfn-init-cmd.log",
-                        "log_stream_name": "${aws:InstanceId}",
+                        "log_stream_name": "{instance_id}",
                         "timestamp_format": "%H: %M: %S%y%b%-d"
                     },
                     {
                         "file_path": "/var/log/cfn-init.log",
                         "log_group_name": "cfn-init.log",
-                        "log_stream_name": "${aws:InstanceId}",
+                        "log_stream_name": "{instance_id}",
                         "timestamp_format": "%H: %M: %S%y%b%-d"
                     },
                     {
                         "file_path": "/var/log/cloud-init-output.log",
                         "log_group_name": "cloud-init-output.log",
-                        "log_stream_name": "${aws:InstanceId}",
+                        "log_stream_name": "{instance_id}",
                         "timestamp_format": "%H: %M: %S%y%b%-d"
                     },
                     {
                         "file_path": "/var/log/cloud-init.log",
                         "log_group_name": "cloud-init.log",
-                        "log_stream_name": "${aws:InstanceId}",
+                        "log_stream_name": "{instance_id}",
                         "timestamp_format": "%H: %M: %S%y%b%-d"
                     }
                 ]
             }
         }
+    },
+    "traces": {
+        "traces_collected": {
+            "xray": {},
+            "otlp": {}
+        }
     }
 }

cdk/Constructs/EKSApplication.cs

@@ -271,7 +271,7 @@ public EKSApplication(Construct scope, string id, IEKSApplicationProps props) :
                         {"namespace", props.Namespace}
                     }},
                     {"data", new Dictionary<string, object>() {
-                        {"cwagentconfig.json", "{\"agent\":{\"omit_hostname\":true},\"logs\":{\"metrics_collected\":{\"emf\":{}}}}"}
+                        {"cwagentconfig.json", "{\"agent\":{\"omit_hostname\":true},\"logs\":{\"metrics_collected\":{\"emf\":{}}},\"traces\":{\"traces_collected\":{\"xray\":{},\"otlp\":{}}}}"}
                     }}
                 }
             });
@@ -363,7 +363,19 @@ public EKSApplication(Construct scope, string id, IEKSApplicationProps props) :
                                             }},
                                             {"ports", new Dictionary<string, object>[] {
                                                 new Dictionary<string, object>() {
-                                                    {"containerPort", 25888},
+                                                    {"containerPort", 25888}, // cloudwatch agent
+                                                    {"protocol", "TCP"}
+                                                },
+                                                new Dictionary<string, object>() {
+                                                    {"containerPort", 2000}, // xray
+                                                    {"protocol", "TCP"}
+                                                },
+                                                new Dictionary<string, object>() {
+                                                    {"containerPort", 4317}, // otlp grpc
+                                                    {"protocol", "TCP"}
+                                                },
+                                                new Dictionary<string, object>() {
+                                                    {"containerPort", 4318}, // otlp http
                                                     {"protocol", "TCP"}
                                                 }
                                             }}