Merge pull request #47 from mmacata/stdout-logging

Enable stdout logging
actinia-org · May 13, 2020 · b683d30 · b683d30
2 parents 2778817 + a529932
commit b683d30
Show file tree

Hide file tree

Showing 35 changed files with 787 additions and 243 deletions.
diff --git a/docker/README.md b/docker/README.md
@@ -60,8 +60,10 @@ docker-compose run --rm --service-ports --entrypoint /bin/sh -v $HOME/repos/acti
 
 docker-compose -f docker/docker-compose-dev.yml run --rm --service-ports --entrypoint /bin/sh -v $HOME/repos/actinia/actinia_core:/src/actinia_core actinia-core
 
+docker-compose -f docker-compose-dev.yml run --rm --entrypoint /bin/bash -v $HOME/repos/actinia/actinia_core/src:/src/actinia_core/src -v $HOME/repos/actinia/actinia_core/scripts:/src/actinia_core/scripts actinia-core
+
 # To avoid cache problems, remove the packaged actinia_core
-pip3 uninstall actinia_core
+pip3 uninstall actinia_core -y
 ```
 Mind the git checkout during build time - even though actinia_core is copied from local sources to the Dockerfile, a git checkout will overwrite changes. During development it is necessary to mount the whole sourcecode.
 
@@ -87,7 +89,7 @@ Alternatively, run the actinia_core server with your mounted source code inside
 cd /src/actinia_core
 python3 setup.py install
 
-bash /src/start-dev.sh
+sh /src/start-dev.sh
 
 # python3 -m actinia_core.main
 gunicorn -b 0.0.0.0:8088 -w 1 --access-logfile=- -k gthread actinia_core.main:flask_app

diff --git a/docker/actinia-core-alpine/Dockerfile b/docker/actinia-core-alpine/Dockerfile
@@ -56,12 +56,12 @@ RUN while IFS=, read -r ADDON SERVER; do unset URL; test -z $SERVER || URL="url=
 # Currently, ADD always breaks cache, as "download_count" increases every time
 # ADD https://api.github.com/repos/mundialis/actinia_statistic_plugin/releases/latest /scratch/actinia_statistic_plugin_latest_release.json
 WORKDIR /build
-# Get statistics plugin (22648768 = 0.0.1)
-RUN curl https://api.github.com/repos/mundialis/actinia_statistic_plugin/releases/22648768 > resp.json && \
+# Get statistics plugin (26188052 = 0.0.2)
+RUN curl https://api.github.com/repos/mundialis/actinia_statistic_plugin/releases/26188052 > resp.json && \
     name=`cat resp.json | jq '.assets[0].name' | tr -d '"'` && \
     cat resp.json | jq '.assets[0].browser_download_url' | xargs curl -L --output /build/$name && rm resp.json
-# Get satellite plugin (22686533 = 0.0.1)
-RUN curl https://api.github.com/repos/mundialis/actinia_satellite_plugin/releases/22686533 > resp.json && \
+# Get satellite plugin (26187974 = 0.0.2)
+RUN curl https://api.github.com/repos/mundialis/actinia_satellite_plugin/releases/26187974 > resp.json && \
     name=`cat resp.json | jq '.assets[0].name' | tr -d '"'` && \
     cat resp.json | jq '.assets[0].browser_download_url' | xargs curl -L --output /build/$name && rm resp.json
 # Get actinia-gdi plugin (25559351 = 0.1.7)
@@ -70,7 +70,8 @@ RUN curl https://api.github.com/repos/mundialis/actinia-gdi/releases/25559351 >
     cat resp.json | jq '.assets[0].browser_download_url' | xargs curl -L --output /build/$name && rm resp.json
 # Install actinia-core and plugins
 COPY --from=build /build/*.whl /build/
-RUN apk add curl && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py pip==20.0.2 && rm get-pip.py
+# the problem with getting broken 20.0.1 seems to be overcome, the outcommenting
+# RUN apk add curl && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py pip==20.0.2 && rm get-pip.py
 RUN pip3 install /build/*
 # See in build stage why we need this here
 RUN pip3 install scikit-learn

diff --git a/docker/actinia-core-alpine/actinia.cfg b/docker/actinia-core-alpine/actinia.cfg
@@ -26,7 +26,7 @@ redis_server_pw = pass
 redis_queue_server_url = redis
 redis_queue_server_port = 6379
 worker_queue_name = actinia_job
-worker_logfile = /actinia_core/workspace/tmp/actinia_worker
+worker_logfile = /actinia_core/workspace/tmp/actinia_worker.log
 
 [LOGGING]
 log_interface = fluentd

diff --git a/docker/actinia-core-dev/Dockerfile b/docker/actinia-core-dev/Dockerfile
@@ -1,13 +1,5 @@
-FROM mundialis/actinia-core
+FROM mundialis/actinia-core:g78-latest-alpine
 
-# Copy actinia config file and start scripts
 COPY actinia.cfg /etc/default/actinia
 
-WORKDIR /src/actinia_core
-RUN python3 setup.py install
-
-EXPOSE 8088
-EXPOSE 9191
-
-ENTRYPOINT ["/bin/bash"]
-CMD ["/src/start.sh"]
+RUN pip3 uninstall actinia-core -y
diff --git a/docker/actinia-core-dev/actinia.cfg b/docker/actinia-core-dev/actinia.cfg
@@ -10,7 +10,7 @@ grass_modules_xml_path = /usr/local/grass7/gui/wxpython/xml/module_items.xml
 grass_default_location = nc_spm_08
 
 [API]
-plugins = ["actinia_statistic_plugin"]
+plugins = ["actinia_statistic_plugin", "actinia_satellite_plugin", "actinia_gdi"]
 force_https_urls = False
 
 [LIMITS]
@@ -26,13 +26,16 @@ redis_server_pw = pass
 redis_queue_server_url = redis
 redis_queue_server_port = 6379
 worker_queue_name = actinia_job
-worker_logfile = /actinia_core/workspace/tmp/actinia_worker
+worker_logfile = /actinia_core/workspace/tmp/actinia_worker.log
 
 [LOGGING]
-log_interface = fluentd
+log_interface = stdout
+log_stdout_format = colored
+log_file_format = colored
+log_stderr_format = plain
 log_fluent_host = fluentd
 log_fluent_port = 24224
-log_level = 1
+log_level = 4
 
 [MISC]
 tmp_workdir = /actinia_core/workspace/tmp

diff --git a/docker/actinia-core-latest/actinia.cfg b/docker/actinia-core-latest/actinia.cfg
@@ -26,7 +26,7 @@ redis_server_pw = pass
 redis_queue_server_url = redis
 redis_queue_server_port = 6379
 worker_queue_name = actinia_job
-worker_logfile = /actinia_core/workspace/tmp/actinia_worker
+worker_logfile = /actinia_core/workspace/tmp/actinia_worker.log
 
 [LOGGING]
 log_interface = fluentd

diff --git a/docker/actinia-core-prod/actinia.cfg b/docker/actinia-core-prod/actinia.cfg
@@ -26,7 +26,7 @@ redis_server_pw = pass
 redis_queue_server_url = redis
 redis_queue_server_port = 6379
 worker_queue_name = actinia_job
-worker_logfile = /actinia_core/workspace/tmp/actinia_worker
+worker_logfile = /actinia_core/workspace/tmp/actinia_worker.log
 
 [LOGGING]
 log_interface = fluentd

diff --git a/docker/actinia-core/actinia.cfg b/docker/actinia-core/actinia.cfg
@@ -26,7 +26,7 @@ redis_server_pw = pass
 redis_queue_server_url = redis
 redis_queue_server_port = 6379
 worker_queue_name = actinia_job
-worker_logfile = /actinia_core/workspace/tmp/actinia_worker
+worker_logfile = /actinia_core/workspace/tmp/actinia_worker.log
 
 [LOGGING]
 log_interface = fluentd

diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml
@@ -1,7 +1,7 @@
 version: "3"
 services:
 
-  actinia-core:
+  actinia:
     build:
       context: actinia-core-dev/
     volumes:
@@ -12,11 +12,14 @@ services:
       - ./actinia-core-data/workspace/temp_db:/actinia_core/workspace/temp_db"
       - ./actinia-core-data/workspace/tmp:/actinia_core/workspace/tmp"
       - ./actinia-core-data/resources:/actinia_core/resources"
-      - ../scripts:/src/actinia_core/scripts
+      - ../.:/src/actinia_core/.
     ports:
       - "8088:8088"
     depends_on:
       - redis
+      # - fluentd
+      # - elasticsearch
+      # - kibana
     cap_add:
       - SYS_PTRACE
 
@@ -36,3 +39,21 @@ services:
     ]
     ports:
         - "6379:6379"
+
+
+  # fluentd:
+  #   build:
+  #     context: fluentd
+  #   image: fluentd
+  #   expose:
+  #     - 24224
+  #
+  # elasticsearch:
+  #   image:  elasticsearch:7.4.0
+  #   environment:
+  #   - discovery.type=single-node
+  #
+  # kibana:
+  #   image: kibana:7.4.0
+  #   ports:
+  #     - "5601:5601"
diff --git a/docker/fluentd/Dockerfile b/docker/fluentd/Dockerfile
@@ -0,0 +1,4 @@
+FROM fluent/fluentd
+RUN ["gem", "install", "fluent-plugin-elasticsearch", "--no-rdoc", "--no-ri", "--version", "3.5.5"]
+RUN rm /fluentd/etc/fluent.conf
+COPY fluent.conf /fluentd/etc
diff --git a/docker/fluentd/fluent.conf b/docker/fluentd/fluent.conf
@@ -0,0 +1,34 @@
+# fluent.conf
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+<match *.**>
+  @type copy
+  <store>
+    @type elasticsearch
+    host elasticsearch
+    port 9200
+    logstash_format true
+    logstash_prefix fluentd
+    logstash_dateformat %Y%m%d
+    include_tag_key true
+    type_name access_log
+    tag_key @log_name
+    flush_interval 1s
+  </store>
+  <store>
+    @type file
+    @id   output1
+    path         /fluentd/log/data.*.log
+    symlink_path /fluentd/log/data.log
+    append       true
+    time_slice_format %Y%m%d
+    time_slice_wait   10m
+    time_format       %Y%m%dT%H%M%S%z
+  </store>
+</match>
+<filter **>
+  @type stdout
+</filter>