Vinicius Temóteo Ferrari

.gitignore

@@ -0,0 +1,93 @@
+HELP.md
+/target/
+!.mvn/wrapper/maven-wrapper.jar
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+/build/
+
+### Infra ###
+/infra/kafka-connect/users.csv
+## FRONT ###
+/front/node_modules
+
+# Logs
+/front/logs
+/front/*.log
+/front/npm-debug.log*
+/front/yarn-debug.log*
+/front/yarn-error.log*
+
+# Runtime data
+pids
+/front/*.pid
+/front/*.seed
+/front/*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+/front/lib-cov
+
+# Coverage directory used by tools like istanbul
+/front/coverage
+
+# nyc test coverage
+/front/.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+/front/.grunt
+
+# Bower dependency directory (https://bower.io/)
+/front/bower_components
+
+# node-waf configuration
+/front/.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+/front/build/Release
+
+# Dependency directories
+/front/node_modules/
+/front/jspm_packages/
+
+# Typescript v1 declaration files
+/front/typings/
+
+# Optional npm cache directory
+/front/.npm
+
+# Optional eslint cache
+/front/.eslintcache
+
+# Optional REPL history
+/front/.node_repl_history
+
+# Output of 'npm pack'
+/front/*.tgz
+
+# Yarn Integrity file
+/front/.yarn-integrity
+
+# dotenv environment variables file
+/front/.env
+
+# next.js build output
+/front/.next

.mvn/wrapper/MavenWrapperDownloader.java

@@ -0,0 +1,114 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Properties;
+
+public class MavenWrapperDownloader {
+
+    /**
+     * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
+     */
+    private static final String DEFAULT_DOWNLOAD_URL =
+            "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar";
+
+    /**
+     * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
+     * use instead of the default one.
+     */
+    private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
+            ".mvn/wrapper/maven-wrapper.properties";
+
+    /**
+     * Path where the maven-wrapper.jar will be saved to.
+     */
+    private static final String MAVEN_WRAPPER_JAR_PATH =
+            ".mvn/wrapper/maven-wrapper.jar";
+
+    /**
+     * Name of the property which should be used to override the default download url for the wrapper.
+     */
+    private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
+
+    public static void main(String args[]) {
+        System.out.println("- Downloader started");
+        File baseDirectory = new File(args[0]);
+        System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
+
+        // If the maven-wrapper.properties exists, read it and check if it contains a custom
+        // wrapperUrl parameter.
+        File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
+        String url = DEFAULT_DOWNLOAD_URL;
+        if(mavenWrapperPropertyFile.exists()) {
+            FileInputStream mavenWrapperPropertyFileInputStream = null;
+            try {
+                mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
+                Properties mavenWrapperProperties = new Properties();
+                mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
+                url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
+            } catch (IOException e) {
+                System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
+            } finally {
+                try {
+                    if(mavenWrapperPropertyFileInputStream != null) {
+                        mavenWrapperPropertyFileInputStream.close();
+                    }
+                } catch (IOException e) {
+                    // Ignore ...
+                }
+            }
+        }
+        System.out.println("- Downloading from: : " + url);
+
+        File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
+        if(!outputFile.getParentFile().exists()) {
+            if(!outputFile.getParentFile().mkdirs()) {
+                System.out.println(
+                        "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'");
+            }
+        }
+        System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
+        try {
+            downloadFileFromURL(url, outputFile);
+            System.out.println("Done");
+            System.exit(0);
+        } catch (Throwable e) {
+            System.out.println("- Error downloading");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private static void downloadFileFromURL(String urlString, File destination) throws Exception {
+        URL website = new URL(urlString);
+        ReadableByteChannel rbc;
+        rbc = Channels.newChannel(website.openStream());
+        FileOutputStream fos = new FileOutputStream(destination);
+        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+        fos.close();
+        rbc.close();
+    }
+
+}

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip

Dockerfile

@@ -0,0 +1,7 @@
+FROM openjdk:11-jre-slim
+
+COPY infra/start.sh /opt/start.sh
+RUN chmod +x /opt/start.sh
+COPY target/trabalhe*.jar /opt/app.jar
+EXPOSE 8080
+CMD /opt/start.sh

README.md

@@ -1,3 +1,78 @@
+# Tecnologias
+
+* Java 11
+* Spring boot 2.1.3
+* Kafka
+* ElasticSearch
+* Docker
+* React
+
+# Endpoint
+
+ | Metodo | URI | Status |
+ |--------| ----|--------|
+ | GET | /v1/users | 200 |
+
+# QueryParms
+
+ | Nome    | Tipo    | Obrigatório |
+ | ------- | ------- | ----------- |
+ | keyword | string  | Sim         |
+ | page    | integer | Não         |
+ | size    | integer | Não         |
+
+# Exemplo de execução
+
+ | User  | Pass  | 
+ | ----- | ----- |
+ | admin | admin |
+```bash
+$ curl -H 'Authorization: Basic YWRtaW46YWRtaW4=' http://localhost:8080/v1/users?keyword=adr&page=0&size=200
+``` 
+
+# Front-end
+Para executar a pagina de teste é necessario acessar pelo endereço ```http://localhost:8080/index.html``` e digitar o login e senha no popup que vai abrir
+
+
+
+# Como executar
+
+Basta executar o arquivo ```.start.sh``` que tudo rodara automaticamente.
+
+## Sistemas testado
+ * Linux ubuntu 14,16,18
+ * Linux ec2 da aws
+ * Linux ubuntu server
+ * Mac OSx
+
+ *Linux RedHat7, openSuse apresentou instabilidade.
+
+# Importante 
+O projeto usa vários programas pesados para atender os requisitos de performance do teste, portanto é aconselhável rodar em um ambiente Linux
+
+
+### Mac OSx
+
+O docker no Mac faz a emulação do ambiente portanto é importante alocar uma parte da memória e processador para o docker, o projeto exige uma quantidade rasurável de recursos do sistema.
+
+Para o teste foi usado um Mac 2013, i5 - 8gb ram e ssd. A configuração ficou em 7164Mb ram e 4 cores reservado para o docker 
+
+### Linux
+
+Foram executados dois testes, um em um i7 e 16Gb ram e outro em um i5 com 8Gb ram.
+
+O primeiro rodou tudo em aproximadamente 40 minutos com toda a carga do Elastic, o segundo demorou mais de 2 horas
+
+
+
+```É possivel executar os testes enquando o banco é populado```
+# Fonte 
+* [Escrevendo 1MM/sec](https://medium.appbase.io/benchmarking-elasticsearch-1-million-writes-per-sec-bf37e7ca8a4c)
+* [Kafka connect](https://medium.appbase.io/benchmarking-elasticsearch-1-million-writes-per-sec-bf37e7ca8a4c)
+
+
+---
+
 ![PicPay](https://user-images.githubusercontent.com/1765696/26998603-711fcf30-4d5c-11e7-9281-0d9eb20337ad.png)
 
 # Teste Backend

infra/docker-compose.yml

@@ -0,0 +1,61 @@
+version: '3'
+services:
+  app:
+    build:
+      context: ../
+      dockerfile: Dockerfile
+    ports:
+      - "8080:8080"
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+    depends_on:
+      - elasticsearch
+  kafka:
+    build:
+      context: ./kafka-docker
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    depends_on:
+      - zookeeper
+  manager:
+    image: hlebalbau/kafka-manager:stable
+    environment:
+      ZK_HOSTS: zookeeper:2181
+    ports:
+      - "9000:9000"
+    depends_on:
+      - zookeeper
+      - kafka
+  connect:
+    build:
+      context: ./kafka-connect
+    ports:
+      - "8086:8086"
+    depends_on:
+      - zookeeper
+      - kafka
+  connect-ui:
+    image: landoop/kafka-connect-ui
+    ports:
+      - "8000:8000"
+    environment:
+      CONNECT_URL: http://connect:8086
+    depends_on:
+      - zookeeper
+      - kafka
+      - manager
+      - connect
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.4.3
+    ports:
+      - "9200:9200"
+    container_name: elasticsearch
+    environment:
+      - discovery.type=single-node

infra/kafka-connect/Dockerfile

@@ -0,0 +1,31 @@
+FROM infra_kafka
+
+COPY worker.properties /tmp/worker.properties
+COPY distributed-worker.properties /tmp/distributed-worker.properties
+COPY file-stream.properties /tmp/file-stream.properties
+COPY elastic-sink.properties /tmp/elastic-sink.properties
+COPY relevant-list1.properties /tmp/relevant-list1.properties
+COPY relevant-list2.properties /tmp/relevant-list2.properties
+COPY kafka-connect-spooldir.tar.gz /tmp/plugins/kafka-connect-spooldir.tar.gz
+COPY kafka-connect-elasticsearch.tar.gz /tmp/plugins/kafka-connect-elasticsearch.tar.gz
+COPY lista_relevancia_1.txt /tmp/lista_relevancia_1.txt
+COPY lista_relevancia_2.txt /tmp/lista_relevancia_2.txt
+#Setup
+RUN mkdir -p /tmp/plugins/kafka-connect-spooldir \
+ && mkdir -p /tmp/plugins/kafka-connect-elasticsearch \
+ && tar -xzvf /tmp/plugins/kafka-connect-spooldir.tar.gz -C /tmp/plugins/kafka-connect-spooldir/ \
+ && tar -xzvf /tmp/plugins/kafka-connect-elasticsearch.tar.gz -C /tmp/plugins/kafka-connect-elasticsearch/ \
+ && curl https://s3.amazonaws.com/careers-picpay/users.csv.gz --output /tmp/users.csv.gz \
+ && gzip -d /tmp/users.csv.gz \
+ && mkdir -p /tmp/in \
+ && mkdir -p /tmp/out \
+ && mkdir -p /tmp/err \
+ && mv /tmp/users.csv /tmp/in/users.csv
+#Topic config
+#CMD connect-distributed.sh /tmp/distributed-worker.properties
+CMD connect-standalone.sh \
+  /tmp/worker.properties \
+  /tmp/file-stream.properties \
+  /tmp/elastic-sink.properties \
+  /tmp/relevant-list1.properties \
+  /tmp/relevant-list2.properties