Skip to content

Commit

Permalink
Merge pull request #18 from devgateway/develop
Browse files Browse the repository at this point in the history
develop into main
  • Loading branch information
mpostelnicu authored Nov 9, 2023
2 parents e04ee15 + 0217d52 commit 82442a2
Showing 61 changed files with 669 additions and 556 deletions.
5 changes: 3 additions & 2 deletions .env
Original file line number Diff line number Diff line change
@@ -8,5 +8,6 @@ INFOBIP_KEY=452789534289
SMSGATEWAY_KEY=98459432897
SMTP_PORT=25
SMTP_HOST=host.docker.internal
RUN_MODE=admin
SERVER_URL=http://localhost:8090
SERVER_URL=http://localhost:8090
SPRING_DEVTOOLS_RESTART_ENABLED=false
DISABLE_EMAIL_SENDING=false
18 changes: 18 additions & 0 deletions .github/workflows/docker-image.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
name: Docker Image CI

on:
push:
branches: [ "develop" ]
pull_request:
branches: [ "develop" ]

jobs:

build:

runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- name: Build the Docker image
run: docker compose build
25 changes: 21 additions & 4 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
FROM maven:3.6-openjdk-8 as compiler
FROM maven:3.8-openjdk-17 as compiler
WORKDIR /tmp/build
RUN microdnf install zip
COPY forms/pom.xml forms/pom.xml
COPY persistence/pom.xml persistence/pom.xml
COPY persistence-mongodb/pom.xml persistence-mongodb/pom.xml
COPY web/pom.xml web/pom.xml
COPY pom.xml .
ENV MAVEN_OPTS="-XX:-TieredCompilation -XX:TieredStopAtLevel=1"
#we run dependency:go-offline on all pom.xml files copied, to get the dependencies first
RUN --mount=type=cache,target=/root/.m2,id=ocnandi-m2 \
find . -name pom.xml \
@@ -13,14 +15,14 @@ RUN --mount=type=cache,target=/root/.m2,id=ocnandi-m2 \
COPY . .
#we compile the code then we explode the fat jar. This is useful to create a reusable layer and save image space/compile time
RUN --mount=type=cache,target=/root/.m2,id=ocnandi-m2 \
mvn -T 4 clean package -DskipTests -Dcheckstyle.skip -Dmaven.gitcommitid.skip=true
mvn -T 1C clean package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dmaven.gitcommitid.skip=true
RUN mkdir -p forms/target/deps \
&& cd forms/target/deps \
&& unzip -qo '../*.jar' || \
( e=$? && if [ $e -ne 1 ]; then exit $e; fi ) \
&& rm -f ../*.*

FROM openjdk:8-jdk-slim as prod
FROM openjdk:17-jdk-slim as prod
WORKDIR /opt/app
RUN apt-get update && apt-get install -y fontconfig libfreetype6 && rm -rf /var/lib/apt/lists/*
#we copy artifacts from exploded jar, one by one, each COPY command will create a separate docker layer
@@ -32,4 +34,19 @@ COPY --chmod=0755 entrypoint.sh .
EXPOSE 8090
ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.12.0/wait /wait
RUN chmod +x /wait
CMD /wait && /opt/app/entrypoint.sh
CMD /wait && /opt/app/entrypoint.sh admin

FROM openjdk:17-jdk-slim as dev
WORKDIR /opt/app
RUN apt-get update && apt-get install -y fontconfig libfreetype6 && rm -rf /var/lib/apt/lists/*
COPY --from=compiler /tmp/build/forms/target/deps/BOOT-INF/lib lib
RUN rm -f lib/persistence*-SNAPSHOT.jar
RUN rm -f lib/web*-SNAPSHOT.jar
COPY --chmod=0755 entrypoint.sh .
EXPOSE 8090
EXPOSE 8000
ENV JAVA_TOOL_OPTIONS -agentlib:jdwp=transport=dt_socket,address=*:8000,server=y,suspend=n
#we use the docker-compose-wait script to wait for port 5432. postgres may be busy importing the db if db is not present
ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.12.0/wait /wait
RUN chmod +x /wait
CMD /wait && /opt/app/entrypoint.sh admin-dev
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -2,6 +2,8 @@

[![Build Status](https://travis-ci.org/devgateway/oc-explorer.svg?branch=develop)](https://travis-ci.org/devgateway/oc-explorer)

[![Docker Image CI](https://github.com/devgateway/ocportal-nandi/actions/workflows/docker-image.yml/badge.svg?branch=develop)](https://github.com/devgateway/ocportal-nandi/actions/workflows/docker-image.yml)

## Presentation

OCE is a project that allows importing the public procurement data, available in the common MS Excel format, into a native [Open Contracting Data Standard (OCDS)](http://standard.open-contracting.org/) NoSQL storage, and then run visual data analytics (display a *live* dashboard with charts, maps and data tables as well as custom comparison charts). Since the data is natively stored in the OCDS format, it can be readily exported in this format without any transformation required, and with great throughput.
51 changes: 29 additions & 22 deletions docker-compose-dev.yml
Original file line number Diff line number Diff line change
@@ -3,12 +3,12 @@ version: '3.8'
services:

app:
image: ocportal-nandi/admin/prod:${TAG-local}
restart: always
image: ocportal-nandi/admin/dev:${TAG-local}
restart: 'no'
extra_hosts:
- "host.docker.internal:host-gateway" #required so docker connects to host interface (this is for sending emails)
build:
target: prod
target: dev
depends_on:
- db
- mongo
@@ -17,6 +17,13 @@ services:
- "8000:8000"
env_file:
- .env
#entrypoint: /bin/sh -c "while sleep 1000; do :; done"
volumes:
- ./forms/target/classes:/opt/app/forms/classes
- ./persistence/target/classes:/opt/app/persistence/classes
- ./persistence-mongodb/target/classes:/opt/app/persistence-mongodb/classes
- ./web/target/classes:/opt/app/web/classes
- ./ui:/opt/app/ui

db:
image: postgis/postgis:11-3.3
@@ -39,25 +46,25 @@ services:
env_file:
- .env

pgbackups:
image: prodrigestivill/postgres-backup-local
restart: 'no'
volumes:
- /opt/pgbackups:/backups
env_file:
- .env
depends_on:
- db
environment:
- POSTGRES_HOST=db
- POSTGRES_DB=ocportal
- POSTGRES_USER=$POSTGRES_USER
- POSTGRES_PASSWORD=$POSTGRES_PASSWORD
- SCHEDULE=@daily
- BACKUP_KEEP_DAYS=30
- BACKUP_KEEP_WEEKS=12
- BACKUP_KEEP_MONTHS=24
- HEALTHCHECK_PORT=5432
# pgbackups:
# image: prodrigestivill/postgres-backup-local
# restart: 'no'
# volumes:
# - /opt/pgbackups:/backups
# env_file:
# - .env
# depends_on:
# - db
# environment:
# - POSTGRES_HOST=db
# - POSTGRES_DB=ocportal
# - POSTGRES_USER=$POSTGRES_USER
# - POSTGRES_PASSWORD=$POSTGRES_PASSWORD
# - SCHEDULE=@daily
# - BACKUP_KEEP_DAYS=30
# - BACKUP_KEEP_WEEKS=12
# - BACKUP_KEEP_MONTHS=24
# - HEALTHCHECK_PORT=5432

volumes:
pgdata:
28 changes: 23 additions & 5 deletions entrypoint.sh
Original file line number Diff line number Diff line change
@@ -7,8 +7,20 @@ COMMON_JAVA_ARGS="$(tr '\n' ' ' <<-EOF
-Dwicket.configuration=deployment
-Dfile.encoding=UTF-8
-DserverURL=$SERVER_URL
-DdisableEmailSending=$DISABLE_EMAIL_SENDING
-Dspring.devtools.restart.enabled=$SPRING_DEVTOOLS_RESTART_ENABLED
-Xms512m
-Xmx4096m
-Xmx6144m
--add-opens=java.naming/javax.naming=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.lang.ref=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.security=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.management/javax.management=ALL-UNNAMED
-Dspring.mail.port=$SMTP_PORT
-Dspring.mail.host=$SMTP_HOST
-XX:MaxMetaspaceSize=512m
@@ -26,7 +38,7 @@ COMMON_JAVA_ARGS="$(tr '\n' ' ' <<-EOF
EOF
)"

case "$RUN_MODE" in
case "$1" in
admin)
JAVA_ARGS="${COMMON_JAVA_ARGS} $(tr '\n' ' ' <<-EOF
-cp .:lib/*
@@ -35,13 +47,19 @@ admin)
)"
exec java $JAVA_ARGS $@
;;
admin-debug)
admin-dev)
JAVA_ARGS="${COMMON_JAVA_ARGS} $(tr '\n' ' ' <<-EOF
-cp .:lib/*
-Dspring.devtools.restart.additional-exclude=logs/**,META-INF/**,ehcache-disk-store/**
-Dspring.devtools.restart.poll-interval=3s
-Dspring.devtools.restart.quiet-period=2s
-XX:+TieredCompilation
-XX:TieredStopAtLevel=1
-noverify
-cp forms/classes:persistence-mongodb/classes:web/classes:persistence/classes:lib/*
org.devgateway.toolkit.forms.wicket.FormsWebApplication
EOF
)"
exec java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 $JAVA_ARGS $@
exec java $JAVA_ARGS $@
;;
*)
exec "$@"
21 changes: 8 additions & 13 deletions forms/pom.xml
Original file line number Diff line number Diff line change
@@ -28,16 +28,16 @@

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>1.8</java.version>
<wicket.version>8.9.0</wicket.version>
<wicketstuff.version>8.9.0</wicketstuff.version>
<wicket.bootstrap.version>2.0.12</wicket.bootstrap.version>
<java.version>17</java.version>
<wicket.version>8.15.0</wicket.version>
<wicketstuff.version>8.15.0</wicketstuff.version>
<wicket.bootstrap.version>2.0.15</wicket.bootstrap.version>
<zt.zip.version>1.13</zt.zip.version>
<wicket.webjars.version>2.0.18</wicket.webjars.version>
<wicket.webjars.version>2.0.20</wicket.webjars.version>
<closure.compiler.version>v20200406</closure.compiler.version>
<cglib.version>3.2.12</cglib.version>
<yuicompressor.version>2.4.8</yuicompressor.version>
<javamelody.version>1.86.0</javamelody.version>
<javamelody.version>1.95.0</javamelody.version>
<exclude.devtools>false</exclude.devtools>
</properties>

<dependencies>
@@ -167,12 +167,6 @@
</exclusions>
</dependency>

<dependency>
<groupId>cglib</groupId>
<artifactId>cglib</artifactId>
<version>${cglib.version}</version>
</dependency>

<dependency>
<groupId>com.google.javascript</groupId>
<artifactId>closure-compiler-unshaded</artifactId>
@@ -360,6 +354,7 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<executable>true</executable>
<excludeDevtools>${exclude.devtools}</excludeDevtools>
</configuration>
<version>${spring.boot.version}</version>
<executions>
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@
import nl.dries.wicket.hibernate.dozer.DozerModel;
import nl.dries.wicket.hibernate.dozer.SessionFinder;
import org.hibernate.Session;
import org.hibernate.engine.spi.SessionImplementor;
import org.springframework.stereotype.Component;

import javax.persistence.EntityManager;
@@ -41,7 +42,7 @@ public class SessionFinderService implements SessionFinder {
*/
@Override
public Session getHibernateSession(final Class<?> clazz) {
return em.unwrap(Session.class);
return em.unwrap(SessionImplementor.class);
}

}
10 changes: 10 additions & 0 deletions forms/src/main/resources/argfile.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
--add-opens=java.naming/javax.naming=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.lang.ref=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.security=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.management/javax.management=ALL-UNNAMED
Binary file removed forms/src/main/resources/banner.gif
Binary file not shown.
3 changes: 1 addition & 2 deletions persistence-mongodb/pom.xml
Original file line number Diff line number Diff line change
@@ -23,10 +23,9 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<start-class>org.devgateway.toolkit.persistence.spring.PersistenceApplication</start-class>
<java.version>1.8</java.version>
<java.version>17</java.version>
<json.patch.version>1.13</json.patch.version>
<commons-digester3.version>3.2</commons-digester3.version>
<reflections.version>0.9.11</reflections.version>
</properties>

<parent>
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ private MongoConstants() {


public static final class OCDSSchemes {
public static final String X_KE_INTERNAL_SCHEMA = "X-KE-OCELGEYO-MARAKWET";
public static final String X_KE_INTERNAL_SCHEMA = "X-KE-NANDI";
public static final String X_KE_IFMIS = "x_KE-IFMIS";
public static final String KE_IFMIS = "KE-IFMIS";
public static final String UNCEFACT = "UNCEFACT";
Original file line number Diff line number Diff line change
@@ -33,9 +33,8 @@ public void createMandatoryImportIndexes() {
new Index().on("roles", Direction.ASC));
getTemplate().indexOps(Organization.class).ensureIndex(new Index().on("name", Direction.ASC).unique());
getTemplate().indexOps(DefaultLocation.class).ensureIndex(new Index().on("description", Direction.ASC));
getTemplate().indexOps("fs.files").ensureIndex(new Index().on("md5", Direction.ASC));
getTemplate().indexOps("fs.files").ensureIndex(new CompoundIndexDefinition(
new Document("filename", 1).append("md5", 1)).unique());
new Document("filename", 1).append("metadata.md5", 1)).unique());
getLogger().info("Added mandatory Mongo indexes");
}

Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@
import org.springframework.core.convert.converter.Converter;
import org.springframework.data.convert.ReadingConverter;
import org.springframework.data.convert.WritingConverter;
import org.springframework.data.mongodb.core.convert.CustomConversions;
import org.springframework.data.mongodb.core.convert.MongoCustomConversions;
import org.springframework.data.mongodb.core.geo.GeoJsonPoint;
import org.springframework.util.Assert;
import org.springframework.util.ObjectUtils;
@@ -46,7 +46,7 @@
*/
@SpringBootApplication
@ComponentScan(value = "org.devgateway", excludeFilters = @ComponentScan.Filter(
type = FilterType.ASPECTJ, pattern = "org.devgateway.jocds.ValidatorConfiguration" ))
type = FilterType.ASPECTJ, pattern = "org.devgateway.jocds.ValidatorConfiguration"))
@PropertySource("classpath:/org/devgateway/toolkit/persistence/mongo/application.properties")
@EnableCaching
public class MongoPersistenceApplication {
@@ -56,12 +56,12 @@ public static void main(final String[] args) {
}

@Bean
public CustomConversions customConversions() {
return new CustomConversions(Arrays
.asList(new Object[]{BigDecimalToDoubleConverter.INSTANCE, DoubleToBigDecimalConverter.INSTANCE,
public MongoCustomConversions customConversions() {
return new MongoCustomConversions(Arrays
.asList(BigDecimalToDoubleConverter.INSTANCE, DoubleToBigDecimalConverter.INSTANCE,
DbObjectToGeoJsonPointConverter.INSTANCE,
ZonedDateTimeReadConverter.INSTANCE, ZonedDateTimeWriteConverter.INSTANCE,
URIToStringConverter.INSTANCE}));
URIToStringConverter.INSTANCE));
}

public enum URIToStringConverter implements Converter<URI, String> {
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
package org.devgateway.toolkit.persistence.mongo.spring;

import com.mongodb.MongoClientURI;
import org.springframework.beans.factory.annotation.Autowire;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.mongo.MongoProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
import org.springframework.data.mongodb.core.convert.CustomConversions;
import org.springframework.data.mongodb.core.SimpleMongoClientDbFactory;
import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
import org.springframework.data.mongodb.core.convert.MongoCustomConversions;

/**
* Created by mpostelnicu on 6/12/17.
@@ -25,11 +24,11 @@ public class MongoTemplateConfig {
private MongoProperties properties;

@Autowired
private CustomConversions customConversions;
private MongoCustomConversions customConversions;

@Bean(autowire = Autowire.BY_NAME, name = "mongoTemplate")
public MongoTemplate mongoTemplate() throws Exception {
MongoTemplate template = new MongoTemplate(new SimpleMongoDbFactory(new MongoClientURI(properties.getUri())));
MongoTemplate template = new MongoTemplate(new SimpleMongoClientDbFactory(properties.getUri()));
((MappingMongoConverter) template.getConverter()).setCustomConversions(customConversions);

// CALL THIS MANULLY, so that all the default converters will be registered!
Loading

0 comments on commit 82442a2

Please sign in to comment.