diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..9e95c0a Binary files /dev/null and b/.DS_Store differ diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 43385f8..7b1d9fc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -105,7 +105,13 @@ jobs: - name: drain-staging-space uses: cloud-gov/cg-cli-tools@main with: - command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh + command: | + apt-get install -y jq && + curl -L -o drain-plugin https://github.com/cloudfoundry/cf-drain-cli/releases/download/v2.0.0/cf-drain-cli-linux && + cf install-plugin -f -r drain-plugin && + rm -f drain-plugin && + mkdir -p /root/.cf/ && touch /root/.cf/config.json && + ./create-space-drain.sh management-staging cf_org: gsa-datagov cf_space: staging cf_username: ${{secrets.CF_SERVICE_USER}} @@ -123,7 +129,13 @@ jobs: - name: drain-prod-space uses: cloud-gov/cg-cli-tools@main with: - command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh + command: | + apt-get install -y jq && + curl -L -o drain-plugin https://github.com/cloudfoundry/cf-drain-cli/releases/download/v2.0.0/cf-drain-cli-linux && + cf install-plugin -f -r drain-plugin && + rm -f drain-plugin && + mkdir -p /root/.cf/ && touch /root/.cf/config.json && + ./create-space-drain.sh management cf_org: gsa-datagov cf_space: prod cf_username: ${{secrets.CF_SERVICE_USER}} diff --git a/.github/workflows/dev-deploy.yml b/.github/workflows/dev-deploy.yml new file mode 100644 index 0000000..85047a3 --- /dev/null +++ b/.github/workflows/dev-deploy.yml @@ -0,0 +1,75 @@ +--- +name: dev-deploy + +on: + push: + branches: + - development + workflow_dispatch: + +jobs: + create-cloudgov-services-development-ssb: + name: create services (development-ssb) + environment: development-ssb + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + - name: create services + uses: cloud-gov/cg-cli-tools@main + with: + command: ./create-cloudgov-services.sh + cf_org: gsa-datagov + cf_space: development-ssb + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} + + deploy-development-ssb: + name: deploy (development-ssb) + environment: development-ssb + runs-on: ubuntu-latest + needs: + - create-cloudgov-services-development-ssb + steps: + - name: checkout + uses: actions/checkout@v2 + - name: build dependencies + run: | + chmod -R 777 logstash + PWD=`pwd` make logstash-installation + - name: deploy-logstash + uses: cloud-gov/cg-cli-tools@main + with: + command: cf push --vars-file vars.development-ssb.yml --strategy rolling + cf_org: gsa-datagov + cf_space: development-ssb + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} + - name: smoke test logstash + run: | + sleep 150 # Logstash is very slow to start up + [ "401" = "$(curl -w '%{http_code}' --output /dev/null --silent https://logstash-development-datagov.app.cloud.gov)" ] + + drain-apps-in-development: + name: drain everything in development space + environment: development-ssb + runs-on: ubuntu-latest + needs: + - deploy-development-ssb + steps: + - name: checkout + uses: actions/checkout@v2 + - name: drain-development-space + uses: cloud-gov/cg-cli-tools@main + with: + command: | + apt-get install -y jq && + curl -L -o drain-plugin https://github.com/cloudfoundry/cf-drain-cli/releases/download/v2.0.0/cf-drain-cli-linux && + cf install-plugin -f -r drain-plugin && + rm -f drain-plugin && + mkdir -p /root/.cf/ && touch /root/.cf/config.json && + ./create-space-drain.sh development-ssb + cf_org: gsa-datagov + cf_space: development + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} diff --git a/.gitignore b/.gitignore index a03129a..13920ff 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ logstash/logstash-oss-*-linux-x86_64.tar.gz logstash/plugins.zip logstash/awscliv2.zip +.DS_Store diff --git a/README.md b/README.md index 1a1e308..56ceab8 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,11 @@ Alternatively, you can auto-drain all apps in a given space by targeting that sp cf target -s prod ./create-space-drain.sh +Note: at development space, it will use the logstack-shipper in development-ssb space: + + cf target -s development + ./create-space-drain.sh development-ssb + After a short delay, logs should begin to flow automatically. ## Setup diff --git a/create-cloudgov-services.sh b/create-cloudgov-services.sh index f97018a..31b3fcd 100755 --- a/create-cloudgov-services.sh +++ b/create-cloudgov-services.sh @@ -13,7 +13,7 @@ space=$(cf target | grep space | cut -d : -f 2 | xargs) randpw(){ openssl rand -base64 40 | tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo; } # Only create stuff in production and staging spaces -if [ "$space" = "management" ] || [ "$space" = "management-staging" ]; then +if [ "$space" = "management" ] || [ "$space" = "management-staging" ] || [ "$space" = "development-ssb" ]; then cf service "${app_name}-s3" > /dev/null 2>&1 || cf create-service s3 basic "${app_name}-s3" --wait& cf service "${app_name}-secrets" > /dev/null 2>&1 || cf create-user-provided-service "${app_name}-secrets" -p '{"DRAIN_USER":"'$(randpw)'","DRAIN_PASSWORD":"'$(randpw)'"}' & diff --git a/create-space-drain.sh b/create-space-drain.sh index 609f397..f56a1b9 100755 --- a/create-space-drain.sh +++ b/create-space-drain.sh @@ -39,9 +39,9 @@ cat > manifest.yml << EOF --- applications: - name: ${prefix}-space-drain - instances: 1 + instances: 0 memory: 64M no-route: true EOF -cf drain-space --drain-name "${prefix}-space-drain" "$drain_url" +cf drain-space --drain-name "${prefix}-space-drain" "$drain_url" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1a34921..eb6efa4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,21 +15,14 @@ services: timeout: 10s start_period: 10s ports: - - "127.0.0.1:4566:4566" - - "127.0.0.1:4571:4571" + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range environment: - - SERVICES=${SERVICES-} - DEBUG=${DEBUG-} - - DATA_DIR=${DATA_DIR-} - - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-} - - HOST_TMP_FOLDER=${TMPDIR:-/tmp/}localstack - DOCKER_HOST=unix:///var/run/docker.sock - - AWS_DEFAULT_REGION=us-east-1 volumes: - - "${TMPDIR:-/tmp}/localstack:/tmp/localstack" + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock" - - ./localstack.d:/docker-entrypoint-initaws.d - logstash: depends_on: diff --git a/logstash/logstash.conf b/logstash/logstash.conf old mode 100644 new mode 100755 index 3521a6b..ecf98e4 --- a/logstash/logstash.conf +++ b/logstash/logstash.conf @@ -63,11 +63,52 @@ filter{ # } # mutate { add_field => { "[@metadata][target_index]" => "index-%{+YYYY.MM}" } } + grok { + match => { "message" => "%{GREEDYDATA:raw_message_content}" } + } + grok { + match => { + "message" => '<%{GREEDYDATA:skip} %{TIMESTAMP_ISO8601:skip} %{DATA:host} %{UUID:proc_id} \[%{DATA:instance_info}\] - \[tags@%{GREEDYDATA:skip} instance_id="%{INT:instance_id}" ip="%{IP:ip}" job="%{GREEDYDATA:skip} space_name="%{DATA:space_name}"\] %{GREEDYDATA:log_data}' + tag_on_failure => [] + overwrite => ["message"] + break_on_match => false + } + } + # for message like: catalog-dev.data.gov - [2023-12-15T21:11:49.508628938Z] "GET /0000000 HTTP/1.1" 404 0 21445 "-" "Mozilla/5.0 ..." "127.0.x.x:xxxxx" "10.xx.2.10:xxxxx" x_forwarded_for:"108.xx.xxx.xxx, 64.252.66.xxx, 127.0.x.x" x_forwarded_proto:"https" ... ... b3:xxx + grok { + match => { + "log_data" => '%{HOSTNAME:hostname} - \[%{TIMESTAMP_ISO8601:timestamp}\] "%{WORD:http_method} %{GREEDYDATA:request} %{DATA:http_version}" %{NUMBER:status} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} "%{DATA:http_referer}" "%{DATA:http_user_agent}" %{GREEDYDATA:skip} x_forwarded_for:"%{IP:real_ip}(?:, %{GREEDYDATA:forwarded_ips})*" x_forwarded_proto:%{GREEDYDATA:skip}' + tag_on_failure => [] + overwrite => ["message"] + break_on_match => false + } + } + # for message like: 2023-12-14 20:31:53,839 INFO [ckan.config.middleware.flask_app] 404 /dataset/xxxx render time 0.023 seconds + grok { + match => { + "log_data" => '%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:log_level} \[%{DATA:logger}\] %{NUMBER:status:int} %{URIPATH:request_path} render time %{NUMBER:render_time:float} seconds' + tag_on_failure => [] + overwrite => ["message"] + break_on_match => false + } + } + if ( + ([log_data] and [log_data] =~ /^NginxLog/) + or + ([hostname] and [http_status] and [hostname] =~ /^logstash-/ and [http_status] == "200") + ) { + drop { } + } mutate { add_field => { "[@metadata][NEWRELIC_KEY]" => "${NEWRELIC_LICENSE_KEY:notpresent}" } add_field => { "[@metadata][AWS_S3_PROXY]" => "${AWS_S3_PROXY:notpresent}" } + remove_field => ["skip"] + } + if [space_name] and [space_name] !~ /development/ { + mutate { + remove_field => ["log_data", "raw_message_content"] + } } - } output { diff --git a/manifest.yml b/manifest.yml index f1d39ab..6f54b8c 100644 --- a/manifest.yml +++ b/manifest.yml @@ -5,6 +5,8 @@ applications: APP_NAME: ((app_name)) LS_JAVA_OPTS: ((logstash_java_options)) https_proxy: ((https_proxy)) + NEWRELIC_LICENSE_KEY: ((logstash_newrelic_license_key)) + NEWRELIC_LOG_URI: ((logstash_newrelic_log_uri)) buildpacks: - binary_buildpack health-check-type: process @@ -17,7 +19,4 @@ applications: - route: ((hostname)).((domain)) services: - ((app_name))-s3 - - ((app_name))-secrets -env: - NEWRELIC_LICENSE_KEY: ((logstash_newrelic_license_key)) - NEWRELIC_LOG_URI: ((logstash_newrelic_log_uri)) + - ((app_name))-secrets \ No newline at end of file diff --git a/vars.development-ssb.yml b/vars.development-ssb.yml new file mode 100644 index 0000000..376b191 --- /dev/null +++ b/vars.development-ssb.yml @@ -0,0 +1,20 @@ +--- +# Name prefix for applications in the logstack. +app_name: logstack + +# Options to pass the JVM +logstash_java_options: -Xms512m -Xmx512m +# Number of instances of the Logstash to run. +logstash_instances: 1 +# Amount of memory to provide to the Logstash (Java) application. +logstash_memory: 1024 +# Assign route for the log shipper. This will be used as the logdrain for other +# Cloud Foundry applications. +hostname: logstash-development-datagov +domain: app.cloud.gov +https_proxy: + +# Optional license key for delivering logs to New Relic +# See https://github.com/newrelic/logstash-output-plugin +logstash_newrelic_license_key: +logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1