diff --git a/.gitignore b/.gitignore index 38a776e98..fbc05da3e 100644 --- a/.gitignore +++ b/.gitignore @@ -117,4 +117,9 @@ hs_err_pid* **/resources/public # build dir -target \ No newline at end of file +target + +### Developer's personal properties ### +**/resources/application*-dev-*.properties +**/resources/application*-dev-*.yaml +**/resources/application*-dev-*.yml \ No newline at end of file diff --git a/README.md b/README.md index bd57583c0..1e87593d2 100644 --- a/README.md +++ b/README.md @@ -1 +1,93 @@ -# ArachneCentralAPI \ No newline at end of file +#Arachne Community Edition build and run manual + +##Prerequisites +For building and run the Arachne please install following applications: +- [Apache Maven 3](https://maven.apache.org/download.cgi) +- [JDK 8](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) +- [LibreOffice 6](https://www.libreoffice.org/download/download/) - for running the ArachneCentralAPI only +- [Apache Solr 7](http://lucene.apache.org/solr/downloads.html) +- [Postgres DBMS 9.6+](https://www.postgresql.org/download/windows/) + +####Prepare databases: +Please create ohdsi user and 2 databases: arachne_portal and datanode. That can achieved by running following command in psql console: +``` +create role ohdsi with LOGIN password 'ohdsi'; +create database arachne_portal owner ohdsi; +create database datanode owner ohdsi; +``` + +##Getting sources +Arachne network consists of two applications – Datanode and CentralApi. Sources are located in the github repositories. Please checkout: +[ArachneCentralAPI](https://github.com/OHDSI/ArachneCentralAPI) +[ArachneNodeAPI](https://github.com/OHDSI/ArachneNodeAPI) +The latest released version can be found in the master branch. + +##Solr Configuration +Download solr 7 binaries. +Solr configuration is stored in ArachneCentralAPI/solr-config. Please run command to create and configure cores: +``` +solr start -c && \ + solr create_collection -c users -n arachne-config && \ + solr create_collection -c data-sources -n arachne-config && \ + solr create_collection -c studies -n arachne-config && \ + solr create_collection -c analyses -n arachne-config && \ + solr create_collection -c analysis-files -n arachne-config && \ + solr create_collection -c papers -n arachne-config && \ + solr create_collection -c paper-protocols -n arachne-config && \ + solr create_collection -c paper-files -n arachne-config && \ + solr create_collection -c submissions -n arachne-config && \ + solr create_collection -c insights -n arachne-config && \ + solr create_collection -c result-files -n arachne-config && \ + solr create_collection -c study-files -n arachne-config && \ + solr zk upconfig -n arachne-config -d /$PATHTO/ArachneCentralAPI/solr_config -z localhost:9983 && \ + solr stop -all +``` +Start solr application by running following command in the terminal: +``` +solr start -c && \ + solr zk upconfig -n arachne-config -d /$PATHTO/ArachneCentralAPI/solr_config -z localhost:9983 +``` +Solr console should be available at: http://localhost:8983/solr + + + +####Build ArachneCentralAPI and ArachneNodeApi +Arachne application property files contains few configuration profiles. For this manual we use DEV. Please review available options in the: +- ArachneCentralAPI/src/main/resources +- ArachneNodeAPI/src/main/resources + +1. Open command prompt terminal in the ArachneCentralAPI/ folder and run: +mvn clean package -DskipTests -DskipDocker -P dev + +2. Open command prompt terminal in the ArachneNodeAPI/ folder and run: +mvn clean package -DskipTests -DskipDocker -P dev + +Two artifacts should be created: ArachneCentralAPI/target/portal-exec.jar and ArachneNodeAPI/target/datanode-exec.jar which are spring-boot fat jars and contains all the required dependencies. + +####Start ArachneCentralAPI +Create folder and grant RW access: mkdir -p /var/arachne/files/jcr/workspaces. +Arachne applications expect jasypt encryption for passwords. Please generate values using e.g.: +[jasypt online encoder](https://www.devglan.com/online-tools/jasypt-online-encryption-decryption) + + +Start application using following command: +``` +java -jar ArachneCentralAPI/target/portal-exec.jar --office.home=/usr/lib/libreoffice/ --jasypt.encryptor.password=dummy "--spring.datasource.password=ENC(3b0hKjcVNZjGGLwd85Q+tw==)" "--spring.mail.password=ENC(O8Of4J1ejz9r7tZo05CS/Q==)" --portal.urlWhiteList=https://localhost:8080 +``` +spring.mail.password and spring.mail.username contains dummy values in this example, please replace them with your settings, otherwise send mail functionality will not work. Please encrypt email and database passwords with the same jasypt password. You can do it via [online](https://www.devglan.com/online-tools/jasypt-online-encryption-decryption) or via [jasypt cli](http://www.jasypt.org/cli.html) + +ArachneCentralAPI should be available at: https://localhost:8080 + +####Start ArachneNodeAPI + +Arachne DataNodeAPI application at start registers in ArachneCentralAPI. In current scenario it should be running on https://localhost:8080 + +``` +java -jar target/datanode-exec.jar --datanode.arachneCentral.host=https://localhost --datanode.arachneCentral.port=8080 --jasypt.encryptor.password=dummy "--spring.datasource.password=ENC(3b0hKjcVNZjGGLwd85Q+tw==)" "--spring.mail.password=ENC(O8Of4J1ejz9r7tZo05CS/Q==)" --spring.datasource.url=jdbc:postgresql://localhost:5432/datanode +``` +ArachneNodeAPI should be available at: https://localhost:8880 + + +Supply your mail sender configuration parameters otherwise emails will not work. + +You may override any configuration parameter using “--name=value” spring boot notation. \ No newline at end of file diff --git a/pom.xml b/pom.xml index 334fa2490..c3ba285ee 100644 --- a/pom.xml +++ b/pom.xml @@ -6,27 +6,27 @@ portal com.odysseusinc.arachne - 1.14.0 + 1.15.0 jar org.springframework.boot spring-boot-starter-parent - 1.5.9.RELEASE + 1.5.20.RELEASE - 1.5.4.RELEASE + 1.5.20.RELEASE 4.2.1.RELEASE com.odysseusinc.arachne.portal.PortalStarter UTF-8 1.8 - 0.6.0 + 0.10.5 1.3.0 42.2.1 2.7 - 3.5 + 3.8.1 2.6.3 ${BUILD_NUMBER} ${BUILD_TIMESTAMP} @@ -34,7 +34,7 @@ 4.1.0 5.2.12.Final 2.0.1 - 1.2.1.RELEASE + 1.2.4.RELEASE @@ -111,6 +111,11 @@ flyway-core 4.2.0 + + org.ohdsi + authenticator + 0.0.1-QA + @@ -121,6 +126,11 @@ ${project.version} jar + + com.odysseusinc.arachne + arachne-common-utils + ${project.version} + com.odysseusinc.arachne arachne-no-handler-found-exception-util @@ -189,11 +199,7 @@ spring-security-test ${spring-security.version} - - - - - + org.postgresql postgresql @@ -211,8 +217,20 @@ io.jsonwebtoken - jjwt + jjwt-api + ${jjwt.version} + + + io.jsonwebtoken + jjwt-impl ${jjwt.version} + runtime + + + io.jsonwebtoken + jjwt-jackson + ${jjwt.version} + runtime org.imgscalr @@ -233,7 +251,7 @@ net.lingala.zip4j zip4j - 1.3.2 + 2.1.1 edu.vt.middleware @@ -356,6 +374,10 @@ jericho-html 3.4 + + org.ohdsi + authenticator + @@ -505,6 +527,10 @@ src/test/resources true + + src/test/resources-binary + false + diff --git a/src/main/java/com/odysseusinc/arachne/portal/PortalStarter.java b/src/main/java/com/odysseusinc/arachne/portal/PortalStarter.java index a27b0fb3f..a401a8b14 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/PortalStarter.java +++ b/src/main/java/com/odysseusinc/arachne/portal/PortalStarter.java @@ -34,7 +34,7 @@ import org.springframework.scheduling.annotation.EnableScheduling; @Configuration -@ComponentScan(basePackages = {"com.odysseusinc.arachne.*"}) +@ComponentScan(basePackages = {"com.odysseusinc.arachne.*", "org.ohdsi.authenticator.*"}) @EnableAutoConfiguration @EnableScheduling @EnableJpaRepositories( diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/AnalysisController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/AnalysisController.java index ed084dc87..6a07aca7f 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/AnalysisController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/AnalysisController.java @@ -64,16 +64,6 @@ public class AnalysisController extends BaseAnalysisController files) throws IOExcepti readResource("r/" + RUN_PLP_ANALYSIS_FILE_NAME))); } - protected void attachIncidenceRatesFiles(List files) throws IOException { - - files.add(new MockMultipartFile(RUN_IR_ANALYSIS_FILE_NAME, RUN_IR_ANALYSIS_FILE_NAME, null, - readResource("r/" + RUN_IR_ANALYSIS_FILE_NAME))); - files.add(new MockMultipartFile(CIRCE_JAR, CIRCE_JAR, null, - readResource(JARS_IR_PATH + CIRCE_JAR_RES))); - files.add(new MockMultipartFile(COMMONS_IO_JAR, COMMONS_IO_JAR, null, - readResource(JARS_IR_PATH + COMMONS_IO_JAR_RES))); - files.add(new MockMultipartFile(COMMONS_LANG_JAR, COMMONS_LANG_JAR, null, - readResource(JARS_IR_PATH + COMMONS_LANG_JAR_RES))); - files.add(new MockMultipartFile(JACKSON_JAR, JACKSON_JAR, null, - readResource(JARS_IR_PATH + JACKSON_JAR_RES))); - files.add(new MockMultipartFile("additionalCriteria.sql", "additionalCriteria.sql", null, - readResource(IR_RESOURCES_PATH + "additionalCriteria.sql"))); - files.add(new MockMultipartFile("analysis_summary.sql", "analysis_summary.sql", null, - readResource(IR_RESOURCES_PATH + "analysis_summary.sql"))); - files.add(new MockMultipartFile("delete_strata.sql", "delete_strata.sql", null, - readResource(IR_RESOURCES_PATH + "delete_strata.sql"))); - files.add(new MockMultipartFile("groupQuery.sql", "groupQuery.sql", null, - readResource(IR_RESOURCES_PATH + "groupQuery.sql"))); - files.add(new MockMultipartFile("ir_analysis_query_builder.r", "ir_analysis_query_builder.r", null, - readResource(IR_RESOURCES_PATH + "ir_analysis_query_builder.r"))); - files.add(new MockMultipartFile("ir_dist.sql", "ir_dist.sql", null, - readResource(IR_RESOURCES_PATH + "ir_dist.sql"))); - files.add(new MockMultipartFile("performAnalysis.sql", "performAnalysis.sql", null, - readResource(IR_RESOURCES_PATH + "performAnalysis.sql"))); - files.add(new MockMultipartFile("strata.sql", "strata.sql", null, - readResource(IR_RESOURCES_PATH + "strata.sql"))); - files.add(new MockMultipartFile("strata_rules.sql", "strata_rules.sql", null, - readResource(IR_RESOURCES_PATH + "strata_rules.sql"))); - files.add(new MockMultipartFile("strata_stats.sql", "strata_stats.sql", null, - readResource(IR_RESOURCES_PATH + "strata_stats.sql"))); - } - - @Override protected void attachCohortHeraclesFiles(List files) throws IOException, URISyntaxException { diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/AuthenticationController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/AuthenticationController.java index a9d636c02..f6dca4fa3 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/AuthenticationController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/AuthenticationController.java @@ -22,12 +22,12 @@ package com.odysseusinc.arachne.portal.api.v1.controller; -import com.odysseusinc.arachne.portal.security.TokenUtils; import com.odysseusinc.arachne.portal.security.passwordvalidator.ArachnePasswordValidator; import com.odysseusinc.arachne.portal.service.LoginAttemptService; import com.odysseusinc.arachne.portal.service.PasswordResetService; import com.odysseusinc.arachne.portal.service.ProfessionalTypeService; import com.odysseusinc.arachne.portal.service.UserService; +import org.ohdsi.authenticator.service.Authenticator; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.web.bind.annotation.RestController; @@ -36,7 +36,7 @@ public class AuthenticationController extends BaseAuthenticationController { public AuthenticationController(AuthenticationManager authenticationManager, - TokenUtils tokenUtils, + Authenticator authenticator, UserService userService, UserDetailsService userDetailsService, PasswordResetService passwordResetService, @@ -45,9 +45,8 @@ public AuthenticationController(AuthenticationManager authenticationManager, LoginAttemptService loginAttemptService) { super(authenticationManager, - tokenUtils, + authenticator, userService, - userDetailsService, passwordResetService, passwordValidator, professionalTypeService, diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAchillesController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAchillesController.java index c2d780474..b00166a8c 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAchillesController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAchillesController.java @@ -129,6 +129,7 @@ public JsonResult> list(@PathVariable("id") Long datas return result; } + @ApiOperation("Retrieve Achilles reports for the specified datasource") @RequestMapping(value = "datasource/{id}/reports", method = RequestMethod.GET) public JsonResult> reports( @PathVariable("id") Long datasourceId) throws NotExistException { diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAdminController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAdminController.java index fc38aa845..4ef52481c 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAdminController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAdminController.java @@ -120,7 +120,7 @@ public BaseAdminController(final BaseDataSourceService dataSourceService, final ProfessionalTypeService professionalTypeService, final BaseAdminService adminService, final BaseStudyService studyService, - final BaseAnalysisService analysisService, + final BaseAnalysisService analysisService, final BasePaperService paperService, final BaseTenantService tenantService, final ConverterUtils converterUtils, @@ -178,7 +178,7 @@ public Page getAll( Pageable pageable, UserSearch userSearch) throws UserNotFoundException { - + final Page users = userService.getPage(pageable, userSearch); final DeletableUserWithTenantsListDTO userDtoList = conversionService.convert(users.getContent(), DeletableUserWithTenantsListDTO.class); return new CustomPageImpl<>(userDtoList, new PageRequest(pageable.getPageNumber() - 1, pageable.getPageSize()), users.getTotalElements()); @@ -217,7 +217,7 @@ public void register( @RequestMapping(value = "/api/v1/admin/users/ids", method = RequestMethod.GET) public List getListOfUserIdsByFilter(final UserSearch userSearch) throws UserNotFoundException { - + final List users = userService.getList(userSearch); return users.stream().map(IUser::getId).map(UserIdUtils::idToUuid).collect(Collectors.toList()); } @@ -250,10 +250,12 @@ public JsonResult> getAdmins( return result; } - @RequestMapping(value = "/api/v1/admin/admins/{id}", method = RequestMethod.POST) - public JsonResult addAdminRole(@PathVariable Long id) { + @ApiOperation(value = "Grant admin role to user", hidden = true) + @RequestMapping(value = "/api/v1/admin/admins/{username:.+}", method = RequestMethod.POST) + public JsonResult addAdminRole(@PathVariable String username) { - userService.addUserToAdmins(id); + U user = userService.getByUsername(username); + userService.addUserToAdmins(user.getId()); return new JsonResult<>(NO_ERROR); } @@ -275,13 +277,16 @@ public JsonResult> suggestUsers( return result; } - @RequestMapping(value = "/api/v1/admin/admins/{id}", method = RequestMethod.DELETE) - public JsonResult removeAdminRole(@PathVariable Long id) { + @ApiOperation(value = "Remove admin role from user", hidden = true) + @RequestMapping(value = "/api/v1/admin/admins/{username:.+}", method = RequestMethod.DELETE) + public JsonResult removeAdminRole(@PathVariable String username) { - userService.removeUserFromAdmins(id); + U user = userService.getByUsername(username); + userService.removeUserFromAdmins(user.getId()); return new JsonResult<>(NO_ERROR); } + @ApiOperation(value = "Trigger Solr reindexing for the specified domain", hidden = true) @RequestMapping(value = "/api/v1/admin/{domain}/reindex-solr", method = RequestMethod.POST) public JsonResult reindexSolr(@PathVariable("domain") final String domain) throws IllegalAccessException, NotExistException, NoSuchFieldException, SolrServerException, IOException { @@ -308,7 +313,8 @@ public JsonResult reindexSolr(@PathVariable("domain") final String domain) return new JsonResult<>(NO_ERROR); } - + + @ApiOperation(value = "Run standard operation for the set of users accounts", hidden = true) @RequestMapping(value = "/api/v1/admin/users/batch", method = RequestMethod.POST) public JsonResult doBatchOperation(@RequestBody BatchOperationDTO dto) { diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAnalysisController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAnalysisController.java index e9e1a3ffa..485e4c2a1 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAnalysisController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAnalysisController.java @@ -21,16 +21,6 @@ package com.odysseusinc.arachne.portal.api.v1.controller; -import static com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult.ErrorCode.NO_ERROR; -import static com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult.ErrorCode.PERMISSION_DENIED; -import static com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult.ErrorCode.VALIDATION_ERROR; -import static com.odysseusinc.arachne.portal.util.CommentUtils.getRecentCommentables; -import static com.odysseusinc.arachne.portal.util.HttpUtils.putFileContentToResponse; -import static org.springframework.web.bind.annotation.RequestMethod.DELETE; -import static org.springframework.web.bind.annotation.RequestMethod.GET; -import static org.springframework.web.bind.annotation.RequestMethod.POST; -import static org.springframework.web.bind.annotation.RequestMethod.PUT; - import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import com.odysseusinc.arachne.commons.api.v1.dto.CommonEntityRequestDTO; import com.odysseusinc.arachne.commons.api.v1.dto.OptionDTO; @@ -62,6 +52,7 @@ import com.odysseusinc.arachne.portal.exception.PermissionDeniedException; import com.odysseusinc.arachne.portal.exception.ServiceNotAvailableException; import com.odysseusinc.arachne.portal.exception.ValidationException; +import com.odysseusinc.arachne.portal.exception.ValidationRuntimeException; import com.odysseusinc.arachne.portal.model.Analysis; import com.odysseusinc.arachne.portal.model.AnalysisFile; import com.odysseusinc.arachne.portal.model.AnalysisUnlockRequest; @@ -85,29 +76,6 @@ import com.odysseusinc.arachne.portal.util.ImportedFile; import com.odysseusinc.arachne.portal.util.ZipUtil; import io.swagger.annotations.ApiOperation; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; -import java.security.Principal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import java.util.zip.ZipOutputStream; -import javax.jms.JMSException; -import javax.jms.ObjectMessage; -import javax.servlet.http.HttpServletResponse; -import javax.validation.Valid; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.assertj.core.api.exception.RuntimeIOException; @@ -135,6 +103,42 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.multipart.MultipartFile; +import javax.jms.JMSException; +import javax.jms.ObjectMessage; +import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.security.Principal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import java.util.zip.ZipOutputStream; + +import static com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult.ErrorCode.NO_ERROR; +import static com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult.ErrorCode.PERMISSION_DENIED; +import static com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult.ErrorCode.VALIDATION_ERROR; +import static com.odysseusinc.arachne.portal.util.CommentUtils.getRecentCommentables; +import static com.odysseusinc.arachne.portal.util.HttpUtils.putFileContentToResponse; +import static org.springframework.web.bind.annotation.RequestMethod.DELETE; +import static org.springframework.web.bind.annotation.RequestMethod.GET; +import static org.springframework.web.bind.annotation.RequestMethod.POST; +import static org.springframework.web.bind.annotation.RequestMethod.PUT; + public abstract class BaseAnalysisController analysisModificationLock= new ConcurrentSkipListSet(); + @Value("${datanode.messaging.importTimeout}") private Long datanodeImportTimeout; @@ -308,7 +314,7 @@ public JsonResult> list( throws PermissionDeniedException, NotExistException { JsonResult> result; - IUser user = userService.getByEmail(principal.getName()); + IUser user = userService.getByUsername(principal.getName()); if (user == null) { result = new JsonResult<>(PERMISSION_DENIED); return result; @@ -328,6 +334,8 @@ public JsonResult> listTypes() throws PermissionDeniedException, NotExistException { List analysisOptionDTOs = Arrays.stream(CommonAnalysisType.values()) + .filter(type -> type != CommonAnalysisType.COHORT_HERACLES) // NOTE: Temporary disable Heracles analysis type due to dysfunctional Packrat + .filter(type -> type != CommonAnalysisType.COHORT_PATHWAY) // NOTE: Temporary disable Pathways until it completely implemented .map(type -> new OptionDTO(type.name(), type.getTitle())) .collect(Collectors.toList()); @@ -343,12 +351,28 @@ public JsonResult addCommonEntityToAnalysis(@PathVariable("analysisId") Long ana Principal principal) throws NotExistException, JMSException, IOException, PermissionDeniedException, URISyntaxException { - final IUser user = getUser(principal); - final DataNode dataNode = dataNodeService.getById(entityReference.getDataNodeId()); - final T analysis = analysisService.getById(analysisId); - final DataReference dataReference = dataReferenceService.addOrUpdate(entityReference.getEntityGuid(), dataNode); - final List entityFiles = getEntityFiles(entityReference, dataNode, analysisType); - return doAddCommonEntityToAnalysis(analysis, dataReference, user, analysisType, entityFiles); + if (!analysisModificationLock.add(analysisId)) { + throw new ValidationRuntimeException( + "Analysis import rejected", + Collections.singletonMap( + entityReference.getEntityGuid(), + Collections.singletonList("Another import into this analysis is in progress") + ) + ); + } + + try { + LOGGER.debug("Started import into analysis {}", analysisId); + final IUser user = getUser(principal); + final DataNode dataNode = dataNodeService.getById(entityReference.getDataNodeId()); + final T analysis = analysisService.getById(analysisId); + final DataReference dataReference = dataReferenceService.addOrUpdate(entityReference.getEntityGuid(), dataNode); + final List entityFiles = getEntityFiles(entityReference, dataNode, analysisType); + return doAddCommonEntityToAnalysis(analysis, dataReference, user, analysisType, entityFiles); + } finally { + analysisModificationLock.remove(analysisId); + LOGGER.debug("Completed import into analysis {}", analysisId); + } } protected JsonResult doAddCommonEntityToAnalysis(T analysis, DataReference dataReference, IUser user, @@ -714,15 +738,10 @@ protected List getEntityFiles(DataReferenceDTO entityReference, D if (entityType.equals(CommonAnalysisType.COHORT_HERACLES)) { attachCohortHeraclesFiles(files); } - if (entityType.equals(CommonAnalysisType.INCIDENCE)) { - attachIncidenceRatesFiles(files); - } } return files; } - protected abstract void attachIncidenceRatesFiles(List files) throws IOException; - protected byte[] readResource(final String path) throws IOException { Resource resource = new ClassPathResource(path); diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAuthenticationController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAuthenticationController.java index ffd51f4dd..1912c797c 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAuthenticationController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseAuthenticationController.java @@ -22,8 +22,6 @@ package com.odysseusinc.arachne.portal.api.v1.controller; -import static com.odysseusinc.arachne.portal.api.v1.controller.util.ControllerUtils.emulateEmailSent; - import com.odysseusinc.arachne.commons.api.v1.dto.CommonAuthMethodDTO; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAuthenticationRequest; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAuthenticationResponse; @@ -41,8 +39,6 @@ import com.odysseusinc.arachne.portal.model.DataNode; import com.odysseusinc.arachne.portal.model.IUser; import com.odysseusinc.arachne.portal.model.PasswordReset; -import com.odysseusinc.arachne.portal.model.security.ArachneUser; -import com.odysseusinc.arachne.portal.security.TokenUtils; import com.odysseusinc.arachne.portal.security.passwordvalidator.ArachnePasswordData; import com.odysseusinc.arachne.portal.security.passwordvalidator.ArachnePasswordValidationResult; import com.odysseusinc.arachne.portal.security.passwordvalidator.ArachnePasswordValidator; @@ -52,11 +48,10 @@ import com.odysseusinc.arachne.portal.service.ProfessionalTypeService; import edu.vt.middleware.password.Password; import io.swagger.annotations.ApiOperation; -import java.io.IOException; -import java.security.Principal; -import javax.servlet.http.HttpServletRequest; -import javax.validation.Valid; import org.apache.solr.client.solrj.SolrServerException; +import org.ohdsi.authenticator.model.UserInfo; +import org.ohdsi.authenticator.service.Authenticator; +import org.pac4j.core.credentials.UsernamePasswordCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; @@ -67,12 +62,18 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.AuthenticationException; import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.validation.BindingResult; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import javax.servlet.http.HttpServletRequest; +import javax.validation.Valid; +import java.io.IOException; +import java.security.Principal; + +import static com.odysseusinc.arachne.portal.api.v1.controller.util.ControllerUtils.emulateEmailSent; + public abstract class BaseAuthenticationController extends BaseController { private static final Logger log = LoggerFactory.getLogger(BaseAuthenticationController.class); @@ -81,29 +82,28 @@ public abstract class BaseAuthenticationController extends BaseController login(@RequestBody CommonAuthent checkIfUserBlocked(username); checkIfUserHasTenant(username); authenticate(authenticationRequest); - String token = this.tokenUtils.generateToken(username); + UserInfo userInfo = authenticator.authenticate(authMethod, + new UsernamePasswordCredentials(username, authenticationRequest.getPassword())); + String token = userInfo.getToken(); CommonAuthenticationResponse authenticationResponse = new CommonAuthenticationResponse(token); jsonResult = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR, authenticationResponse); loginAttemptService.loginSucceeded(username); @@ -160,14 +162,16 @@ private JsonResult getJsonResultForUnsuccessfulLog private void checkIfUserBlocked(String username) throws PermissionDeniedException { - if (loginAttemptService.isBlocked(username)) { - throw new PermissionDeniedException("You have exceeded the number of allowed login attempts. Please try again later."); + final Long remainingAccountLockPeriodSeconds = loginAttemptService.getRemainingAccountLockPeriod(username); + if (remainingAccountLockPeriodSeconds != null) { + final String errorMessage = String.format("You have exceeded the number of allowed login attempts. Please try again in %s seconds.", remainingAccountLockPeriodSeconds); + throw new PermissionDeniedException(errorMessage); } } - protected void checkIfUserHasTenant(String email) throws AuthenticationException { + protected void checkIfUserHasTenant(String username) throws AuthenticationException { - IUser user = userService.getByEmailInAnyTenant(email); + IUser user = userService.getByUsernameInAnyTenant(username); if (user == null) { throw new BadCredentialsException(ErrorMessages.BAD_CREDENTIALS.getMessage()); } @@ -196,7 +200,7 @@ public JsonResult logout(HttpServletRequest request) { try { String token = request.getHeader(tokenHeader); if (token != null) { - tokenUtils.addInvalidateToken(token); + authenticator.invalidateToken(token); } result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); result.setResult(true); @@ -216,15 +220,9 @@ public JsonResult refresh(HttpServletRequest request) { JsonResult result; try { String token = request.getHeader(this.tokenHeader); - String username = this.tokenUtils.getUsernameFromToken(token); - ArachneUser user = (ArachneUser) this.userDetailsService.loadUserByUsername(username); - if (this.tokenUtils.canTokenBeRefreshed(token, user.getLastPasswordReset())) { - String refreshedToken = this.tokenUtils.refreshToken(token); - result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); - result.setResult(refreshedToken); - } else { - result = new JsonResult<>(JsonResult.ErrorCode.UNAUTHORIZED); - } + UserInfo userInfo = authenticator.refreshToken(token); + result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); + result.setResult(userInfo.getToken()); } catch (Exception ex) { log.error(ex.getMessage(), ex); result = new JsonResult<>(JsonResult.ErrorCode.UNAUTHORIZED); @@ -261,7 +259,7 @@ public JsonResult resetPassword( if (principal != null) { String token = request.getHeader(tokenHeader); - tokenUtils.addInvalidateToken(token); + authenticator.invalidateToken(token); } JsonResult result; if (binding.hasErrors()) { @@ -293,7 +291,7 @@ public JsonResult resetPassword( public JsonResult info(Principal principal) { final JsonResult result; - IUser user = userService.getByEmailInAnyTenant(principal.getName()); + IUser user = userService.getByUsernameInAnyTenant(principal.getName()); final UserInfoDTO userInfo = conversionService.convert(user, UserInfoDTO.class); result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); result.setResult(userInfo); diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeCommonAnalysisController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeCommonAnalysisController.java index e30daef3b..866013096 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeCommonAnalysisController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeCommonAnalysisController.java @@ -30,6 +30,7 @@ import com.odysseusinc.arachne.portal.model.DataNode; import com.odysseusinc.arachne.portal.service.BaseDataNodeService; import com.odysseusinc.arachne.portal.service.messaging.BaseDataNodeMessageService; +import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; @@ -60,6 +61,7 @@ public BaseDataNodeCommonAnalysisController(BaseDataNodeService baseDataNode * Returns list of cohorts defined in Atlas connected to the Data node * (for Central's UI) */ + @ApiOperation("Returns list of cohorts defined in Atlas connected to the Data node") @RequestMapping( value = "/api/v1/data-nodes/{dataNodeId}/{type}", method = GET diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeController.java index 6acc15d66..46572bfec 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeController.java @@ -234,6 +234,7 @@ private void validate(final C_DS_DTO commonDataSourceDTO) throws BindException { } } + @ApiOperation("List datasources for the specified data node") @GetMapping(value = "/api/v1/data-nodes/{dataNodeId}/data-sources") public List getDataSourcesForDataNode(@PathVariable("dataNodeId") Long dataNodeId) { @@ -244,6 +245,7 @@ public List getDataSourcesForDataNode(@PathVariable("dataNodeId") return converterUtils.convertList(dataSources, DataSourceDTO.class); } + @ApiOperation("Get information about specified Data node") @RequestMapping(value = "/api/v1/data-nodes/{dataNodeId}", method = RequestMethod.GET) public JsonResult getDataNode(@PathVariable("dataNodeId") Long dataNodeId) { @@ -254,6 +256,7 @@ public JsonResult getDataNode(@PathVariable("dataNodeId") Long data return new JsonResult<>(JsonResult.ErrorCode.NO_ERROR, conversionService.convert(dataNode, DataNodeDTO.class)); } + @ApiOperation("List all non-virtual data nodes") @RequestMapping(value = "/api/v1/data-nodes", method = RequestMethod.GET) public List getDataNodes() { @@ -261,6 +264,7 @@ public List getDataNodes() { return converterUtils.convertList(dataNodes, DataNodeDTO.class); } + @ApiOperation("List all accessble for the specified user non-virtual data nodes") @RequestMapping(value = "/api/v1/data-nodes/suggest", method = RequestMethod.GET) public List suggestDataNodes(Principal principal) throws PermissionDeniedException { diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeMessagingController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeMessagingController.java index 862ef967c..e479df93e 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeMessagingController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataNodeMessagingController.java @@ -53,6 +53,8 @@ import javax.jms.JMSException; import javax.jms.ObjectMessage; import javax.validation.Valid; + +import io.swagger.annotations.ApiOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -92,6 +94,7 @@ public BaseDataNodeMessagingController( * Returns pending requests for CommonEntity list * (for polling by Node's back) */ + @ApiOperation("Returns pending requests for CommonEntity list") @RequestMapping( value = "/api/v1/data-nodes/entity-lists/requests", method = GET @@ -108,6 +111,7 @@ public CommonListEntityRequest getListRequests( * Posts responses for for CommonEntity list requests * (for pushing by Node's back) */ + @ApiOperation("Posts responses for for CommonEntity list requests") @RequestMapping( value = "/api/v1/data-nodes/entity-lists/responses", method = POST @@ -133,6 +137,7 @@ public void saveListResponse( } } + @ApiOperation("Create or update Atlas instance entry") @RequestMapping(value = "/api/v1/data-nodes/atlases", method = POST) public AtlasShortDTO createOrUpdateAtlas( @RequestBody @Valid AtlasShortDTO atlasShortDTO, @@ -162,6 +167,7 @@ public AtlasShortDTO createOrUpdateAtlas( return conversionService.convert(updated, AtlasShortDTO.class); } + @ApiOperation("Delete instance entry") @RequestMapping(value = "/api/v1/data-nodes/atlases/{id}", method = DELETE) public void deleteAtlas( @PathVariable("id") Long id @@ -170,6 +176,7 @@ public void deleteAtlas( atlasService.delete(id); } + @ApiOperation("List all EntityRequests") @RequestMapping( value = "/api/v1/data-nodes/entities", method = GET @@ -200,6 +207,7 @@ public List getEntityRequests( return cohortRequests; } + @ApiOperation("Save entity") @RequestMapping( value = "/api/v1/data-nodes/common-entity/{id}", method = POST diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataSourceController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataSourceController.java index b5b037e0c..197126d21 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataSourceController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseDataSourceController.java @@ -25,10 +25,10 @@ import static com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult.ErrorCode.NO_ERROR; import com.odysseusinc.arachne.commons.api.v1.dto.CommonBaseDataSourceDTO; -import com.odysseusinc.arachne.commons.api.v1.dto.CommonCDMVersionDTO; import com.odysseusinc.arachne.commons.api.v1.dto.CommonDataSourceDTO; import com.odysseusinc.arachne.commons.api.v1.dto.OptionDTO; import com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult; +import com.odysseusinc.arachne.commons.types.CommonCDMVersionDTO; import com.odysseusinc.arachne.commons.types.DBMSType; import com.odysseusinc.arachne.portal.api.v1.dto.FacetedSearchResultDTO; import com.odysseusinc.arachne.portal.api.v1.dto.IDataSourceDTO; @@ -87,6 +87,7 @@ public BaseDataSourceController(GenericConversionService conversionService, this.studyDataSourceService = studyDataSourceService; } + @ApiOperation("Update and publish datasource") @RequestMapping(value = "/api/v1/data-sources/{id}", method = RequestMethod.PUT) public JsonResult update( Principal principal, @@ -114,7 +115,6 @@ private JsonResult updateDataSource(Principal principal, Long dataSourceId, if (bindingResult.hasErrors()) { result = setValidationErrors(bindingResult); } else { - IUser user = getUser(principal); final DS exist = dataSourceService.getByIdInAnyTenant(dataSourceId); DS dataSource = convertDTOToDataSource(commonDataSourceDTO); dataSource.setId(dataSourceId); @@ -129,6 +129,7 @@ private JsonResult updateDataSource(Principal principal, Long dataSourceId, return result; } + @ApiOperation("Update datasource information") @RequestMapping(value = "/api/v1/data-sources/{id}/from-node", method = RequestMethod.PUT) public JsonResult updateFieldsDefinedAtNode( @PathVariable("id") Long dataSourceId, @@ -144,6 +145,7 @@ public JsonResult updateFieldsDefinedAtNode( return result; } + @ApiOperation("Suggest datasources for study") @RequestMapping(value = "/api/v1/data-sources/search-data-source", method = RequestMethod.GET) public JsonResult> suggestDataSource(Principal principal, @RequestParam("studyId") Long studyId, @@ -166,6 +168,7 @@ public JsonResult> suggestDataSource(Principal principal, protected abstract Class getDataSourceDTOClass(); + @ApiOperation("Suggest datasources") @RequestMapping(value = "/api/v1/data-sources", method = RequestMethod.GET) public JsonResult list(Principal principal, @ModelAttribute SearchDataCatalogDTO searchDTO @@ -177,6 +180,7 @@ public JsonResult list(Principal principal, return new JsonResult<>(NO_ERROR, conversionService.convert(searchResult, getSearchResultClass())); } + @ApiOperation("List accessible for user datasources") @RequestMapping(value = "/api/v1/data-sources/my", method = RequestMethod.GET) public Page getUserDataSources(Principal principal, @RequestParam(name = "query", required = false, defaultValue = "") String query, @@ -192,6 +196,7 @@ public Page getUserDataSources(Principal principal, return new CustomPageImpl<>(dataSourceDTOs, pageRequest, dataSources.getTotalElements()); } + @ApiOperation("Find unsecured datasource by uuid") @RequestMapping(value = "/api/v1/data-sources/byuuid/{uuid}", method = RequestMethod.GET) public JsonResult getByUuid(@PathVariable("uuid") String dataSourceUuid) throws NotExistException { @@ -201,6 +206,7 @@ public JsonResult getByUuid(@PathVariable("uuid") String dataSourceUuid) th return result; } + @ApiOperation("Get datasource by id") @RequestMapping(value = "/api/v1/data-sources/{id}", method = RequestMethod.GET) public JsonResult get(@PathVariable("id") Long dataSourceId) throws NotExistException { @@ -210,6 +216,7 @@ public JsonResult get(@PathVariable("id") Long dataSourceId) throws NotExis return result; } + @ApiOperation("List existing datasource entities by ids") @RequestMapping(value = "/api/v1/data-sources/commondata", method = RequestMethod.GET) public JsonResult> getCommonBaseDataSourceDTOs( @RequestParam("id") List dataSourceIds) throws NotExistException { @@ -256,6 +263,7 @@ public JsonResult publishDataSource(Principal principal, return updateDataSource(principal, dataSourceId, commonDataSourceDTO, bindingResult); } + @ApiOperation("List supported CDM versions") @RequestMapping(value = "/api/v1/data-sources/cdm-versions", method = RequestMethod.GET) public JsonResult> getCDMVersions() { @@ -272,6 +280,7 @@ public JsonResult> getCDMVersions() { protected abstract DS convertDTOToDataSource(DTO dto); + @ApiOperation("Get datasource by id in any tenant") @RequestMapping(value = "/api/v1/data-sources/{id}/complete", method = RequestMethod.GET) public JsonResult getWhole(@PathVariable("id") Long dataSourceId) throws NotExistException { @@ -279,6 +288,7 @@ public JsonResult getWhole(@PathVariable("id") Long dataSourceId) throws return new JsonResult<>(NO_ERROR, conversionService.convert(dataSource, getDataSourceDTOClass())); } + @ApiOperation("List supported DBMS") @RequestMapping(value = "/api/v1/data-sources/dbms-types", method = RequestMethod.GET) public List getDBMSTypes() { diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseExpertFinderController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseExpertFinderController.java index 3563d220e..c03a219fe 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseExpertFinderController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseExpertFinderController.java @@ -87,7 +87,7 @@ public JsonResult viewProfile( Principal principal, @PathVariable("userId") String userId) { - IUser logginedUser = userService.getByEmail(principal.getName()); + IUser logginedUser = userService.getByUsername(principal.getName()); JsonResult result; IUser user = userService.getByUuidAndInitializeCollections(userId); UserProfileDTO userProfileDTO = conversionService.convert(user, UserProfileDTO.class); @@ -103,10 +103,19 @@ public JsonResult get( @PathVariable("id") Long id ) { - JsonResult result; - U user = userService.getByIdInAnyTenant(id); + return buildResponse(userService.getByIdInAnyTenant(id)); + } + + @RequestMapping(value = "/api/v1/user-management/users/byusername/{username:.+}", method = GET) + public JsonResult getByUsername(@PathVariable("username") String username) { + + return buildResponse(userService.getByUsernameInAnyTenant(username)); + } + + private JsonResult buildResponse(U user) { + CommonUserDTO userDTO = conversionService.convert(user, CommonUserDTO.class); - result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); + JsonResult result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); result.setResult(userDTO); return result; } diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseSkillController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseSkillController.java index c1f891f69..abe1110e6 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseSkillController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseSkillController.java @@ -132,7 +132,7 @@ public JsonResult delete(@PathVariable("skillId") Long id) throws NotEx @RequestMapping(value = "/api/v1/user-management/skills", method = RequestMethod.GET) public JsonResult> list(Principal principal) { - Long userId = userService.getByEmail(principal.getName()).getId(); + Long userId = userService.getByUsername(principal.getName()).getId(); JsonResult> result; List skills = skillService.getAllExpectOfUserSkills(userId); result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseStudyController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseStudyController.java index 54a7606d2..d6dc60859 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseStudyController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseStudyController.java @@ -174,7 +174,7 @@ public JsonResult create( throws NotExistException, NotUniqueException { JsonResult result; - IUser user = userService.getByEmail(principal.getName()); + IUser user = userService.getByUsername(principal.getName()); if (user != null) { if (binding.hasErrors()) { result = setValidationErrors(binding); diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseUserController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseUserController.java index c29dcf328..f6a4b4189 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseUserController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/BaseUserController.java @@ -81,7 +81,6 @@ import com.odysseusinc.arachne.portal.model.UserStudy; import com.odysseusinc.arachne.portal.model.search.PaperSearch; import com.odysseusinc.arachne.portal.model.search.StudySearch; -import com.odysseusinc.arachne.portal.security.TokenUtils; import com.odysseusinc.arachne.portal.security.passwordvalidator.ArachnePasswordValidator; import com.odysseusinc.arachne.portal.service.AnalysisUnlockRequestService; import com.odysseusinc.arachne.portal.service.BaseDataNodeService; @@ -94,6 +93,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.security.Principal; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -114,8 +114,10 @@ import org.apache.commons.lang3.math.NumberUtils; import org.apache.http.client.utils.URIBuilder; import org.apache.solr.client.solrj.SolrServerException; +import org.ohdsi.authenticator.service.Authenticator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.convert.support.GenericConversionService; import org.springframework.validation.BindingResult; import org.springframework.validation.FieldError; @@ -144,7 +146,6 @@ public abstract class BaseUserController< private static final String DATA_NODE_NOT_FOUND_EXCEPTION = "dataNode %s not found"; private static final String INVITATION_HOME_PAGE = "/study-manager/studies/"; - protected final TokenUtils tokenUtils; protected final BaseUserService userService; protected final BaseStudyService studyService; protected final GenericConversionService conversionService; @@ -154,9 +155,11 @@ public abstract class BaseUserController< protected final BasePaperService paperService; protected final BaseSubmissionService submissionService; protected final ArachnePasswordValidator passwordValidator; + protected final Authenticator authenticator; + @Value("${arachne.token.header}") + private String tokenHeader; - public BaseUserController(TokenUtils tokenUtils, - BaseUserService userService, + public BaseUserController(BaseUserService userService, BaseStudyService studyService, GenericConversionService conversionService, BaseDataNodeService baseDataNodeService, @@ -164,9 +167,9 @@ public BaseUserController(TokenUtils tokenUtils, AnalysisUnlockRequestService analysisUnlockRequestService, BasePaperService paperService, BaseSubmissionService submissionService, - ArachnePasswordValidator passwordValidator) { + ArachnePasswordValidator passwordValidator, + Authenticator authenticator) { - this.tokenUtils = tokenUtils; this.userService = userService; this.studyService = studyService; this.conversionService = conversionService; @@ -176,6 +179,7 @@ public BaseUserController(TokenUtils tokenUtils, this.paperService = paperService; this.submissionService = submissionService; this.passwordValidator = passwordValidator; + this.authenticator = authenticator; } @ApiOperation("Password restrictions") @@ -250,9 +254,7 @@ public void activate( throws IOException, UserNotFoundException, NotExistException, NoSuchFieldException, IllegalAccessException, SolrServerException, URISyntaxException { - tokenUtils - .getAuthToken(request) - .forEach(token -> tokenUtils.addInvalidateToken(token)); + getAuthToken(request).forEach(authenticator::invalidateToken); Boolean activated; try { @@ -280,7 +282,7 @@ public JsonResult saveUserAvatar( throws IOException, WrongFileFormatException, ValidationException, ImageProcessingException, MetadataException, IllegalAccessException, SolrServerException, NoSuchFieldException { JsonResult result; - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); if (file != null && file.length > 0) { userService.saveAvatar(user, file[0]); } else { @@ -299,7 +301,7 @@ public void getUserAvatar( HttpServletResponse response) throws IOException { final Optional userName = Optional.ofNullable(principal != null ? principal.getName() : null); - U user = userName.map(userService::getByEmail).orElse(null); + U user = userName.map(userService::getByUsername).orElse(null); userService.putAvatarToResponse(response, user); } @@ -327,7 +329,7 @@ public JsonResult saveProfile( NoSuchFieldException { JsonResult result; - IUser owner = userService.getByEmail(principal.getName()); + IUser owner = userService.getByUsername(principal.getName()); if (binding.hasErrors()) { result = setValidationErrors(binding); } else { @@ -349,7 +351,7 @@ public JsonResult changePassword(@RequestBody @Valid ChangePasswordDTO changePas ) throws ValidationException, PasswordValidationException { JsonResult result; - U loggedUser = userService.getByEmail(principal.getName()); + U loggedUser = userService.getByUsername(principal.getName()); try { userService.updatePassword(loggedUser, changePasswordDTO.getOldPassword(), changePasswordDTO.getNewPassword()); result = new JsonResult<>(NO_ERROR); @@ -368,7 +370,7 @@ public JsonResult addSkill( ) throws NotExistException, IllegalAccessException, SolrServerException, IOException, NoSuchFieldException { JsonResult result; - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); user = userService.addSkillToUser(user.getId(), skillId); UserProfileDTO userProfileDTO = conversionService.convert(user, UserProfileDTO.class); result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); @@ -384,7 +386,7 @@ public JsonResult removeSkill( ) throws NotExistException, IllegalAccessException, SolrServerException, IOException, NoSuchFieldException { JsonResult result; - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); user = userService.removeSkillFromUser(user.getId(), skillId); UserProfileDTO userProfileDTO = conversionService.convert(user, UserProfileDTO.class); result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); @@ -407,7 +409,7 @@ public JsonResult addLink( result.getValidatorErrors().put(fieldError.getField(), fieldError.getDefaultMessage()); } } else { - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); user = userService.addLinkToUser(user.getId(), conversionService.convert(userLinkDTO, UserLink.class)); UserProfileDTO userProfileDTO = conversionService.convert(user, UserProfileDTO.class); @@ -425,7 +427,7 @@ public JsonResult removeLink( ) throws NotExistException { JsonResult result; - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); user = userService.removeLinkFromUser(user.getId(), linkId); UserProfileDTO userProfileDTO = conversionService.convert(user, UserProfileDTO.class); result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); @@ -449,7 +451,7 @@ public JsonResult addPublication( } } else { - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); user = userService.addPublicationToUser(user.getId(), conversionService.convert(userPublicationDTO, UserPublication.class)); UserProfileDTO userProfileDTO = conversionService.convert(user, UserProfileDTO.class); @@ -467,7 +469,7 @@ public JsonResult removePublication( ) throws NotExistException { JsonResult result; - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); user = userService.removePublicationFromUser(user.getId(), publicationId); UserProfileDTO userProfileDTO = conversionService.convert(user, UserProfileDTO.class); result = new JsonResult<>(JsonResult.ErrorCode.NO_ERROR); @@ -482,7 +484,7 @@ public JsonResult> invitations( @RequestParam(value = "studyId", required = false) Long studyId ) throws NotExistException { - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); Stream invitationables; if (studyId != null) { @@ -606,7 +608,7 @@ public JsonResult invitationAcceptViaInvitation( @Valid @RequestBody InvitationActionDTO invitationActionDTO ) throws NotExistException, AlreadyExistException, IOException { - U user = userService.getByEmail(principal.getName()); + U user = userService.getByUsername(principal.getName()); return invitationAccept(invitationActionDTO, user); } @@ -733,9 +735,12 @@ public JsonResult linkUserToDataNode(@PathVariable("datanodeSid") Long datanodeI final DN dataNode = Optional.ofNullable(baseDataNodeService.getById(datanodeId)).orElseThrow(() -> new NotExistException(String.format(DATA_NODE_NOT_FOUND_EXCEPTION, datanodeId), DataNode.class)); - final U user = userService.getByUnverifiedEmailInAnyTenant(linkUserToDataNode.getUserName()); + final U user = userService.getByUsernameInAnyTenant(linkUserToDataNode.getUserName()); + if (Objects.isNull(user)) { + throw new NotExistException(String.format("User %s not found", linkUserToDataNode.getUserName()), IUser.class); + } baseDataNodeService.linkUserToDataNode(dataNode, user); - return new JsonResult(NO_ERROR); + return new JsonResult<>(NO_ERROR); } @ApiOperation("Unlink User to DataNode") @@ -762,7 +767,7 @@ public JsonResult> relinkAllUsersToDataNode(@PathVariable("d final DN dataNode = baseDataNodeService.getById(dataNodeId); final Set users = linkUserToDataNodes.stream() - .map(link -> new DataNodeUser(userService.getByUnverifiedEmailInAnyTenant(link.getUserName()), dataNode)) + .map(link -> new DataNodeUser(userService.getByUsernameInAnyTenant(link.getUserName()), dataNode)) .collect(Collectors.toSet()); baseDataNodeService.relinkAllUsersToDataNode(dataNode, users); @@ -821,4 +826,26 @@ public void updateUser( userService.setActiveTenant(user, dto.getActiveTenantId()); } } + + protected List getAuthToken(HttpServletRequest request) { + + List tokens = new ArrayList<>(); + + // Get token from header + String headerToken = request.getHeader(tokenHeader); + if (headerToken != null) { + tokens.add(headerToken); + } + + // Get token from cookie + if (request.getCookies() != null) { + Arrays.stream(request.getCookies()) + .filter(cookie -> cookie.getName().equalsIgnoreCase(tokenHeader) && cookie.getValue() != null) + .findFirst() + .ifPresent(cookie -> tokens.add(cookie.getValue())); + } + + return tokens; + } + } diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/GlobalSearchController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/GlobalSearchController.java index dbdffe120..ac2f1d957 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/GlobalSearchController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/GlobalSearchController.java @@ -61,7 +61,7 @@ public GlobalSearchResultDTO list( final @ModelAttribute SearchGlobalDTO searchDto) throws NotExistException, SolrServerException, NoSuchFieldException, IOException, PermissionDeniedException { - final IUser user = userService.getByEmail(principal.getName()); + final IUser user = userService.getByUsername(principal.getName()); if (user == null) { throw new PermissionDeniedException(); diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/RoleController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/RoleController.java index 3f8100092..0e24a0f81 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/RoleController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/RoleController.java @@ -30,7 +30,6 @@ import com.odysseusinc.arachne.portal.exception.ValidationException; import com.odysseusinc.arachne.portal.model.Role; import com.odysseusinc.arachne.portal.service.RoleService; -import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import java.util.LinkedList; import java.util.List; @@ -46,11 +45,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import springfox.documentation.annotations.ApiIgnore; -/** - * Created by AKrutov on 19.10.2016. - */ -@Api(hidden = true) +@ApiIgnore @Validated @RestController public class RoleController extends BaseController { @@ -61,7 +58,7 @@ public class RoleController extends BaseController { private RoleService roleService; - @ApiOperation(value = "Register new role.", hidden = true) + @ApiOperation(value = "Register new role.") @RequestMapping(value = "/api/v1/admin/roles", method = RequestMethod.POST) public JsonResult create( @RequestBody @Valid RoleDTO roleDTO, @@ -84,7 +81,7 @@ public JsonResult create( } - @ApiOperation(value = "Get role description.", hidden = true) + @ApiOperation(value = "Get role description.") @RequestMapping(value = "/api/v1/admin/roles/{roleId}", method = RequestMethod.GET) public JsonResult get(@PathVariable("roleId") Long id) throws NotExistException { @@ -100,7 +97,7 @@ public JsonResult get(@PathVariable("roleId") Long id) throws NotExistE return result; } - @ApiOperation(value = "Edit role.", hidden = true) + @ApiOperation(value = "Edit role.") @RequestMapping(value = "/api/v1/admin/roles/{roleId}", method = RequestMethod.PUT) public JsonResult update(@PathVariable("roleId") Long id, @RequestBody @Valid RoleDTO roleDTO, BindingResult binding) throws NotExistException, NotUniqueException, ValidationException { @@ -123,7 +120,7 @@ public JsonResult update(@PathVariable("roleId") Long id, @RequestBody return result; } - @ApiOperation(value = "Delete role.", hidden = true) + @ApiOperation(value = "Delete role.") @RequestMapping(value = "/api/v1/admin/roles/{roleId}", method = RequestMethod.DELETE) public JsonResult delete(@PathVariable("roleId") Long id) throws NotExistException { @@ -139,7 +136,7 @@ public JsonResult delete(@PathVariable("roleId") Long id) throws NotExi return result; } - @ApiOperation(value = "List roles.", hidden = true) + @ApiOperation(value = "List roles.") @RequestMapping(value = "/api/v1/admin/roles", method = RequestMethod.GET) public JsonResult> list() { diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/UserController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/UserController.java index b9892c3c6..7622c4782 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/UserController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/UserController.java @@ -26,7 +26,6 @@ import com.odysseusinc.arachne.portal.api.v1.dto.UserProfileGeneralDTO; import com.odysseusinc.arachne.portal.model.Analysis; import com.odysseusinc.arachne.portal.model.DataNode; -import com.odysseusinc.arachne.portal.model.DataSource; import com.odysseusinc.arachne.portal.model.IDataSource; import com.odysseusinc.arachne.portal.model.IUser; import com.odysseusinc.arachne.portal.model.Paper; @@ -37,7 +36,6 @@ import com.odysseusinc.arachne.portal.model.User; import com.odysseusinc.arachne.portal.model.search.PaperSearch; import com.odysseusinc.arachne.portal.model.search.StudySearch; -import com.odysseusinc.arachne.portal.security.TokenUtils; import com.odysseusinc.arachne.portal.security.passwordvalidator.ArachnePasswordValidator; import com.odysseusinc.arachne.portal.service.AnalysisUnlockRequestService; import com.odysseusinc.arachne.portal.service.DataNodeService; @@ -46,6 +44,7 @@ import com.odysseusinc.arachne.portal.service.UserService; import com.odysseusinc.arachne.portal.service.analysis.AnalysisService; import com.odysseusinc.arachne.portal.service.submission.SubmissionService; +import org.ohdsi.authenticator.service.Authenticator; import org.springframework.core.convert.support.GenericConversionService; import org.springframework.web.bind.annotation.RestController; @@ -54,8 +53,7 @@ public class UserController extends BaseUserController { - public UserController(TokenUtils tokenUtils, - UserService userService, + public UserController(UserService userService, StudyService studyService, GenericConversionService conversionService, DataNodeService dataNodeService, @@ -63,11 +61,11 @@ public UserController(TokenUtils tokenUtils, AnalysisUnlockRequestService analysisUnlockRequestService, PaperService paperService, SubmissionService submissionService, - ArachnePasswordValidator passwordValidator + ArachnePasswordValidator passwordValidator, + Authenticator authenticator ) { - super(tokenUtils, - userService, + super( userService, studyService, conversionService, dataNodeService, @@ -75,7 +73,8 @@ public UserController(TokenUtils tokenUtils, analysisUnlockRequestService, paperService, submissionService, - passwordValidator); + passwordValidator, + authenticator); } @Override diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/submission/BaseSubmissionController.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/submission/BaseSubmissionController.java index 21982361d..fbc4c8d46 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/submission/BaseSubmissionController.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/controller/submission/BaseSubmissionController.java @@ -64,6 +64,7 @@ import com.odysseusinc.arachne.storage.model.ArachneFileMeta; import com.odysseusinc.arachne.storage.service.ContentStorageService; import io.swagger.annotations.ApiOperation; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Path; @@ -76,6 +77,7 @@ import javax.servlet.http.HttpServletResponse; import javax.validation.Valid; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -86,6 +88,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.multipart.MultipartFile; public abstract class BaseSubmissionController extends BaseController { @@ -125,7 +128,7 @@ public JsonResult> createSubmission( if (principal == null) { throw new PermissionDeniedException(); } - IUser user = userService.getByEmail(principal.getName()); + IUser user = userService.getByUsername(principal.getName()); if (user == null) { throw new PermissionDeniedException(); } @@ -189,7 +192,7 @@ public JsonResult approveSubmissionResult( approveDTO.setIsSuccess(true); Submission updatedSubmission = submissionService.approveSubmissionResult(submissionId, approveDTO, userService - .getByEmail(principal.getName())); + .getByUsername(principal.getName())); DTO submissionDTO = conversionService.convert(updatedSubmission, getSubmissionDTOClass()); return new JsonResult<>(NO_ERROR, submissionDTO); @@ -202,24 +205,27 @@ public JsonResult approveSubmissionResult( public JsonResult uploadSubmissionResults( Principal principal, @RequestParam("submissionId") Long id, + @RequestParam(value = "archive", defaultValue = "false") boolean archive, @Valid UploadFileDTO uploadFileDTO - ) throws IOException, NotExistException, ValidationException { + ) throws IOException, NotExistException { LOGGER.info("uploading result files for submission with id='{}'", id); - - JsonResult.ErrorCode errorCode; - Boolean hasResult; if (uploadFileDTO.getFile() == null) { - errorCode = JsonResult.ErrorCode.VALIDATION_ERROR; - hasResult = false; + return new JsonResult<>(JsonResult.ErrorCode.VALIDATION_ERROR, false); + } + MultipartFile file = uploadFileDTO.getFile(); + String uploadFileName = ObjectUtils.firstNonNull(uploadFileDTO.getLabel(), file.getOriginalFilename()); + + File tempFile = File.createTempFile(uploadFileName, ""); + tempFile.deleteOnExit(); + file.transferTo(tempFile); + + if (archive) { + submissionService.uploadResultsByDataOwner(id, tempFile); } else { - submissionService.uploadResultsByDataOwner(id, uploadFileDTO.getLabel(), uploadFileDTO.getFile()); - errorCode = JsonResult.ErrorCode.NO_ERROR; - hasResult = true; + submissionService.uploadResultsByDataOwner(id, uploadFileName, tempFile); } - JsonResult result = new JsonResult<>(errorCode); - result.setResult(hasResult); - return result; + return new JsonResult<>(JsonResult.ErrorCode.NO_ERROR, true); } @ApiOperation("Delete manually uploaded submission result file") @@ -279,7 +285,7 @@ public void downloadAllSubmissionResultFiles( "attachment; filename=" + archiveName); Submission submission = submissionService.getSubmissionById(submissionId); - IUser user = userService.getByEmail(principal.getName()); + IUser user = userService.getByUsername(principal.getName()); submissionService .getSubmissionResultAllFiles(user, submission.getSubmissionGroup().getAnalysis().getId(), submissionId, archiveName, response.getOutputStream()); @@ -343,7 +349,7 @@ public List getResultFiles( @RequestParam(value = "real-name", required = false) String realName ) throws PermissionDeniedException, IOException { - IUser user = userService.getByEmail(principal.getName()); + IUser user = userService.getByUsername(principal.getName()); ResultFileSearch resultFileSearch = new ResultFileSearch(); resultFileSearch.setPath(path); @@ -456,7 +462,7 @@ private ArachneFileMeta getResultFile(Principal principal, Long submissionId, St throws NotExistException, PermissionDeniedException { Submission submission = submissionService.getSubmissionById(submissionId); - IUser user = userService.getByEmail(principal.getName()); + IUser user = userService.getByUsername(principal.getName()); return submissionService.getResultFileAndCheckPermission(user, submission, submission.getSubmissionGroup().getAnalysis().getId(), fileUuid); } diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/BaseSubmissionDTO.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/BaseSubmissionDTO.java index c8c955ebc..de4dbfbfa 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/BaseSubmissionDTO.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/BaseSubmissionDTO.java @@ -12,7 +12,7 @@ public class BaseSubmissionDTO extends DTO { private PermissionsDTO permissions; private Date createdAt; private ShortUserDTO author; - private Map resultInfo; + private Object resultInfo; private Boolean hidden; public BaseSubmissionDTO() { @@ -86,11 +86,11 @@ public void setAuthor(ShortUserDTO author) { this.author = author; } - public Map getResultInfo() { + public Object getResultInfo() { return resultInfo; } - public void setResultInfo(Map resultInfo) { + public void setResultInfo(Object resultInfo) { this.resultInfo = resultInfo; } diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/CreateStudyDTO.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/CreateStudyDTO.java index 173de76d3..9d83b12c3 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/CreateStudyDTO.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/CreateStudyDTO.java @@ -38,7 +38,7 @@ public class CreateStudyDTO { }) private String title; - @NotNull + @NotNull(message = "Study type should be defined") private Long typeId; public String getTitle() { diff --git a/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/converters/submission/BaseSubmissionToBaseSubmissionDTOConverter.java b/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/converters/submission/BaseSubmissionToBaseSubmissionDTOConverter.java index d5ce44293..fd9171df7 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/converters/submission/BaseSubmissionToBaseSubmissionDTOConverter.java +++ b/src/main/java/com/odysseusinc/arachne/portal/api/v1/dto/converters/submission/BaseSubmissionToBaseSubmissionDTOConverter.java @@ -23,16 +23,27 @@ package com.odysseusinc.arachne.portal.api.v1.dto.converters.submission; import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.odysseusinc.arachne.portal.api.v1.dto.*; +import com.google.gson.JsonElement; +import com.google.gson.reflect.TypeToken; +import com.odysseusinc.arachne.portal.api.v1.dto.BaseSubmissionDTO; +import com.odysseusinc.arachne.portal.api.v1.dto.DataSourceDTO; +import com.odysseusinc.arachne.portal.api.v1.dto.PermissionsDTO; +import com.odysseusinc.arachne.portal.api.v1.dto.SubmissionStatusDTO; +import com.odysseusinc.arachne.portal.api.v1.dto.SubmissionStatusHistoryElementDTO; import com.odysseusinc.arachne.portal.api.v1.dto.converters.BaseConversionServiceAwareConverter; import com.odysseusinc.arachne.portal.exception.NotExistException; -import com.odysseusinc.arachne.portal.model.*; +import com.odysseusinc.arachne.portal.model.IDataSource; +import com.odysseusinc.arachne.portal.model.Submission; +import com.odysseusinc.arachne.portal.model.SubmissionStatus; +import com.odysseusinc.arachne.portal.model.SubmissionStatusHistoryElement; import com.odysseusinc.arachne.portal.model.security.ArachneUser; import com.odysseusinc.arachne.portal.util.DataNodeUtils; -import org.springframework.security.core.context.SecurityContextHolder; - +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Objects; +import org.springframework.security.core.context.SecurityContextHolder; public abstract class BaseSubmissionToBaseSubmissionDTOConverter extends BaseConversionServiceAwareConverter { @@ -66,9 +77,19 @@ public DTO convert(T source) { } dto.setPermissions(conversionService.convert(source, PermissionsDTO.class)); proceedAdditionalFields(dto, source); - final JsonObject resultInfo = source.getResultInfo(); - final Map map = new Gson().fromJson(resultInfo, Map.class); - dto.setResultInfo(map); + final JsonElement resultInfo = source.getResultInfo(); + Gson gson = new Gson(); + Object result = null; + if (Objects.nonNull(resultInfo)) { + if (resultInfo.isJsonObject()) { + result = gson.fromJson(resultInfo, Map.class); + } else if (resultInfo.isJsonArray()) { + Type type = new TypeToken>>() { + }.getType(); + result = gson.fromJson(resultInfo, type); + } + } + dto.setResultInfo(result); dto.setHidden(source.getHidden()); return dto; } diff --git a/src/main/java/com/odysseusinc/arachne/portal/config/AntivirusConfig.java b/src/main/java/com/odysseusinc/arachne/portal/config/AntivirusConfig.java index 1c3353f5e..edc49a46b 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/config/AntivirusConfig.java +++ b/src/main/java/com/odysseusinc/arachne/portal/config/AntivirusConfig.java @@ -59,7 +59,7 @@ public RetryTemplate antivirusRetryTemplate(AntivirusProperties properties) { AntivirusProperties.RetryConfig retryConfig = properties.getRetry(); Map, Boolean> retryableExceptions = new HashMap<>(); retryableExceptions.put(CommunicationException.class, true); - RetryPolicy policy = new ExpressionRetryPolicy(retryConfig.getMaxAttempts(), retryableExceptions, true, "#{message.contains('Error while communicating with the server')}"); + RetryPolicy policy = new ExpressionRetryPolicy(retryConfig.getMaxAttempts(), retryableExceptions, true, "#{message.contains('Error while communicating with the server')}",true); retryTemplate.setRetryPolicy(policy); AntivirusProperties.BackOffPolicyConfig backOffPolicyConfig = retryConfig.getBackoff(); ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); diff --git a/src/main/java/com/odysseusinc/arachne/portal/config/SwaggerConfig.java b/src/main/java/com/odysseusinc/arachne/portal/config/SwaggerConfig.java index e15cdad1e..86188579e 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/config/SwaggerConfig.java +++ b/src/main/java/com/odysseusinc/arachne/portal/config/SwaggerConfig.java @@ -23,7 +23,6 @@ package com.odysseusinc.arachne.portal.config; import com.odysseusinc.arachne.commons.config.DocketWrapper; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; @@ -37,29 +36,27 @@ @ConditionalOnProperty(prefix = "swagger", name = "enable") public class SwaggerConfig { + @Value("${project.version}") + private String projectVersion; + @Value("${arachne.token.header}") private String arachneTokenHeader; - @Autowired - private DocketWrapper docketWrapper; - @Bean - public Docket api() { - + public Docket docket(DocketWrapper docketWrapper) { return docketWrapper.getDocket(); } @Bean public DocketWrapper docketWrapper() { - - return new DocketWrapper("Arachne Central", + return DocketWrapper.createDocketWrapper( + "Arachne Central", "Arachne Central API", - "1.0.0", + projectVersion, "", arachneTokenHeader, RestController.class, "com.odysseusinc.arachne.portal.api.v1.controller" ); } - } diff --git a/src/main/java/com/odysseusinc/arachne/portal/config/WebSecurityConfig.java b/src/main/java/com/odysseusinc/arachne/portal/config/WebSecurityConfig.java index 6af9db7ec..58c915a37 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/config/WebSecurityConfig.java +++ b/src/main/java/com/odysseusinc/arachne/portal/config/WebSecurityConfig.java @@ -43,6 +43,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.PostConstruct; +import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -53,11 +54,11 @@ import org.springframework.beans.factory.BeanInitializationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.ClassPathResource; import org.springframework.http.HttpMethod; -import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; @@ -186,19 +187,15 @@ public ArachnePasswordValidator passwordValidator() throws IOException { } @Bean - @Override - public AuthenticationManager authenticationManagerBean() throws Exception { + public AuthenticationTokenFilter authenticationTokenFilterBean() throws Exception { - return super.authenticationManagerBean(); + return new AuthenticationTokenFilter(); } - @Bean - public AuthenticationTokenFilter authenticationTokenFilterBean() throws Exception { + public FilterRegistrationBean authenticationTokenFilterRegistration(AuthenticationTokenFilter filter) { - AuthenticationTokenFilter authenticationTokenFilter = new AuthenticationTokenFilter(); - authenticationTokenFilter.setAuthenticationManager(authenticationManagerBean()); - return authenticationTokenFilter; + return disableFilter(filter); } @Bean @@ -207,6 +204,20 @@ public AuthenticationSystemTokenFilter authenticationSystemTokenFilter() { return new AuthenticationSystemTokenFilter(baseDataNodeService); } + @Bean + public FilterRegistrationBean authenticationSystemTokenFilterRegistration(AuthenticationSystemTokenFilter filter) { + + return disableFilter(filter); + } + + private FilterRegistrationBean disableFilter(Filter filter) { + + FilterRegistrationBean registrationBean = new FilterRegistrationBean(); + registrationBean.setFilter(filter); + registrationBean.setEnabled(false); + return registrationBean; + } + @Bean public LoginRequestFilter loginRequestFilter() { diff --git a/src/main/java/com/odysseusinc/arachne/portal/config/WebSocketConfig.java b/src/main/java/com/odysseusinc/arachne/portal/config/WebSocketConfig.java index 3fe9f58dd..4978ab664 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/config/WebSocketConfig.java +++ b/src/main/java/com/odysseusinc/arachne/portal/config/WebSocketConfig.java @@ -24,8 +24,8 @@ import com.odysseusinc.arachne.portal.model.IUser; import com.odysseusinc.arachne.portal.model.User; -import com.odysseusinc.arachne.portal.security.TokenUtils; import com.odysseusinc.arachne.portal.service.BaseUserService; +import org.ohdsi.authenticator.service.Authenticator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -69,15 +69,15 @@ public class WebSocketConfig extends AbstractWebSocketMessageBrokerConfigurer { private String tokenHeader; private BaseUserService userService; - private TokenUtils tokenUtils; + private Authenticator authenticator; private DefaultSimpUserRegistry userRegistry = new DefaultSimpUserRegistry(); private DefaultUserDestinationResolver resolver = new DefaultUserDestinationResolver(userRegistry); @Autowired public WebSocketConfig(@Lazy BaseUserService userService, - TokenUtils tokenUtils) { + Authenticator authenticator) { this.userService = userService; - this.tokenUtils = tokenUtils; + this.authenticator = authenticator; } @Bean @@ -123,11 +123,11 @@ public Message preSend(Message message, MessageChannel channel) { List tokenList = accessor.getNativeHeader(tokenHeader); if (tokenList != null && tokenList.size() > 0) { String authToken = tokenList.get(0).toString(); - String username = tokenUtils.getUsernameFromToken(authToken); - if (!tokenUtils.isExpired(authToken)) { + String username = authenticator.resolveUsername(authToken); + if (authenticator.refreshToken(authToken) != null) { IUser user = userService.getByUsername(username); if (user != null) { - principal = () -> user.getUsername(); + principal = user::getUsername; accessor.setUser(principal); } } diff --git a/src/main/java/com/odysseusinc/arachne/portal/config/properties/AchillesProperties.java b/src/main/java/com/odysseusinc/arachne/portal/config/properties/AchillesProperties.java index b137699f7..9aee79605 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/config/properties/AchillesProperties.java +++ b/src/main/java/com/odysseusinc/arachne/portal/config/properties/AchillesProperties.java @@ -29,52 +29,14 @@ @Validated public class AchillesProperties { - private Executor executor = new Executor(); - - public Executor getExecutor() { + private DefaultExecutorConfigValues executor = new DefaultExecutorConfigValues(); + public DefaultExecutorConfigValues getExecutor() { return executor; } - public void setExecutor(Executor executor) { - + public void setExecutor(DefaultExecutorConfigValues executor) { this.executor = executor; } - public static class Executor { - - private Integer corePoolSize = 1; - private Integer maxPoolSize = 2; - private Integer queueCapacity = 10; - - public Integer getCorePoolSize() { - - return corePoolSize; - } - - public void setCorePoolSize(Integer corePoolSize) { - - this.corePoolSize = corePoolSize; - } - - public Integer getMaxPoolSize() { - - return maxPoolSize; - } - - public void setMaxPoolSize(Integer maxPoolSize) { - - this.maxPoolSize = maxPoolSize; - } - - public Integer getQueueCapacity() { - - return queueCapacity; - } - - public void setQueueCapacity(Integer queueCapacity) { - - this.queueCapacity = queueCapacity; - } - } } diff --git a/src/main/java/com/odysseusinc/arachne/portal/config/properties/AntivirusProperties.java b/src/main/java/com/odysseusinc/arachne/portal/config/properties/AntivirusProperties.java index c430ca97a..039c1a73d 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/config/properties/AntivirusProperties.java +++ b/src/main/java/com/odysseusinc/arachne/portal/config/properties/AntivirusProperties.java @@ -28,16 +28,14 @@ @ConfigurationProperties(prefix = "antivirus") @Validated public class AntivirusProperties { - private AchillesProperties.Executor executor = new AchillesProperties.Executor(); + private DefaultExecutorConfigValues executor = new DefaultExecutorConfigValues(); private RetryConfig retry = new RetryConfig(); - public AchillesProperties.Executor getExecutor() { - + public DefaultExecutorConfigValues getExecutor() { return executor; } - public void setExecutor(AchillesProperties.Executor executor) { - + public void setExecutor(DefaultExecutorConfigValues executor) { this.executor = executor; } @@ -46,42 +44,6 @@ public RetryConfig getRetry() { return retry; } - public static class Executor { - - private Integer corePoolSize = 1; - private Integer maxPoolSize = 2; - private Integer queueCapacity = 10; - - public Integer getCorePoolSize() { - - return corePoolSize; - } - - public void setCorePoolSize(Integer corePoolSize) { - - this.corePoolSize = corePoolSize; - } - - public Integer getMaxPoolSize() { - - return maxPoolSize; - } - - public void setMaxPoolSize(Integer maxPoolSize) { - - this.maxPoolSize = maxPoolSize; - } - - public Integer getQueueCapacity() { - - return queueCapacity; - } - - public void setQueueCapacity(Integer queueCapacity) { - - this.queueCapacity = queueCapacity; - } - } public static class RetryConfig { private int maxAttempts = 5; diff --git a/src/main/java/com/odysseusinc/arachne/portal/config/properties/DefaultExecutorConfigValues.java b/src/main/java/com/odysseusinc/arachne/portal/config/properties/DefaultExecutorConfigValues.java new file mode 100644 index 000000000..fb39bb121 --- /dev/null +++ b/src/main/java/com/odysseusinc/arachne/portal/config/properties/DefaultExecutorConfigValues.java @@ -0,0 +1,60 @@ +/* + * + * Copyright 2018 Odysseus Data Services, inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Company: Odysseus Data Services, Inc. + * Product Owner/Architecture: Gregory Klebanov + * Authors: Ing. Alexandr Cumarav + * Created: May 23, 2019 + * + */ + +package com.odysseusinc.arachne.portal.config.properties; + +public class DefaultExecutorConfigValues { + + private Integer corePoolSize = 1; + private Integer maxPoolSize = 2; + private Integer queueCapacity = 10; + + public Integer getCorePoolSize() { + + return corePoolSize; + } + + public void setCorePoolSize(Integer corePoolSize) { + + this.corePoolSize = corePoolSize; + } + + public Integer getMaxPoolSize() { + + return maxPoolSize; + } + + public void setMaxPoolSize(Integer maxPoolSize) { + + this.maxPoolSize = maxPoolSize; + } + + public Integer getQueueCapacity() { + + return queueCapacity; + } + + public void setQueueCapacity(Integer queueCapacity) { + + this.queueCapacity = queueCapacity; + } +} diff --git a/src/main/java/com/odysseusinc/arachne/portal/db/migration/V20190923174625__IncidenceRateResultInfo.java b/src/main/java/com/odysseusinc/arachne/portal/db/migration/V20190923174625__IncidenceRateResultInfo.java new file mode 100644 index 000000000..2c4fcf047 --- /dev/null +++ b/src/main/java/com/odysseusinc/arachne/portal/db/migration/V20190923174625__IncidenceRateResultInfo.java @@ -0,0 +1,63 @@ +package com.odysseusinc.arachne.portal.db.migration; + +import com.google.gson.JsonElement; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import com.odysseusinc.arachne.commons.config.flyway.ApplicationContextAwareSpringMigration; +import com.odysseusinc.arachne.portal.model.Analysis; +import com.odysseusinc.arachne.portal.model.Submission; +import com.odysseusinc.arachne.portal.repository.submission.SubmissionRepository; +import com.odysseusinc.arachne.portal.service.analysis.BaseAnalysisService; +import com.odysseusinc.arachne.portal.util.SubmissionHelper; +import java.util.List; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallbackWithoutResult; +import org.springframework.transaction.support.TransactionTemplate; + +@Component +public class V20190923174625__IncidenceRateResultInfo implements ApplicationContextAwareSpringMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V20190923174625__IncidenceRateResultInfo.class); + + private final BaseAnalysisService analysisService; + private final SubmissionRepository submissionRepository; + private final SubmissionHelper submissionHelper; + private final TransactionTemplate transactionTemplate; + + public V20190923174625__IncidenceRateResultInfo(BaseAnalysisService analysisService, + SubmissionRepository submissionRepository, + SubmissionHelper submissionHelper, + TransactionTemplate transactionTemplate) { + + this.analysisService = analysisService; + this.submissionRepository = submissionRepository; + this.submissionHelper = submissionHelper; + this.transactionTemplate = transactionTemplate; + } + + @Override + public void migrate() throws Exception { + + List analyses = analysisService.findByType(CommonAnalysisType.INCIDENCE); + LOGGER.info("Migrate Incidence Rate analyses ResultInfo, found: {} analyses", analyses.size()); + transactionTemplate.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(TransactionStatus transactionStatus) { + analyses.forEach(analysis -> { + List submissions = analysis.getSubmissions(); + submissions.forEach(s -> { + JsonElement resultInfo = s.getResultInfo(); + if (Objects.nonNull(resultInfo) && resultInfo.isJsonObject()) { + submissionHelper.updateSubmissionExtendedInfo(s); + submissionRepository.save(s); + } + }); + }); + } + }); + LOGGER.info("Incidence Rate ResultInfo migrated"); + } +} \ No newline at end of file diff --git a/src/main/java/com/odysseusinc/arachne/portal/model/ArachneFile.java b/src/main/java/com/odysseusinc/arachne/portal/model/ArachneFile.java index b0ec5117a..ad69a3f09 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/model/ArachneFile.java +++ b/src/main/java/com/odysseusinc/arachne/portal/model/ArachneFile.java @@ -22,6 +22,7 @@ package com.odysseusinc.arachne.portal.model; +import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; import com.odysseusinc.arachne.portal.security.ArachnePermission; import com.odysseusinc.arachne.portal.security.HasArachnePermissions; import com.odysseusinc.arachne.storage.model.ArachneFileMeta; @@ -61,7 +62,7 @@ public ArachneFile(String uuid, String label, String realName, String contentTyp this.uuid = uuid; this.label = label; - this.realName = realName; + this.realName = CommonFilenameUtils.sanitizeFilename(realName); this.contentType = contentType; this.created = created; this.updated = updated; @@ -80,17 +81,17 @@ public void setUuid(String uuid) { @Deprecated public String getRealName() { - return realName; + return getName(); } public String getName() { - return realName; + return CommonFilenameUtils.sanitizeFilename(realName); } public void setRealName(String realName) { - this.realName = realName; + this.realName = CommonFilenameUtils.sanitizeFilename(realName); } public Date getCreated() { diff --git a/src/main/java/com/odysseusinc/arachne/portal/model/BaseDataSource.java b/src/main/java/com/odysseusinc/arachne/portal/model/BaseDataSource.java index dcbeb3b9e..4197ec102 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/model/BaseDataSource.java +++ b/src/main/java/com/odysseusinc/arachne/portal/model/BaseDataSource.java @@ -22,9 +22,9 @@ package com.odysseusinc.arachne.portal.model; -import com.odysseusinc.arachne.commons.api.v1.dto.CommonCDMVersionDTO; import com.odysseusinc.arachne.commons.api.v1.dto.CommonHealthStatus; import com.odysseusinc.arachne.commons.api.v1.dto.CommonModelType; +import com.odysseusinc.arachne.commons.types.CommonCDMVersionDTO; import com.odysseusinc.arachne.commons.types.DBMSType; import com.odysseusinc.arachne.portal.api.v1.dto.converters.DataSourceSolrExtractors; import com.odysseusinc.arachne.portal.model.security.Tenant; @@ -151,7 +151,7 @@ public boolean equals(final Object obj) { if (this == obj) { return true; } - if (obj == null || !(obj instanceof BaseDataSource)) { + if (!(obj instanceof BaseDataSource)) { return false; } final BaseDataSource s = (BaseDataSource) obj; diff --git a/src/main/java/com/odysseusinc/arachne/portal/model/BaseUser.java b/src/main/java/com/odysseusinc/arachne/portal/model/BaseUser.java index 72e531f30..7f6362f86 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/model/BaseUser.java +++ b/src/main/java/com/odysseusinc/arachne/portal/model/BaseUser.java @@ -605,4 +605,5 @@ public void setActiveTenant(Tenant activeTenant) { this.activeTenant = activeTenant; } + } diff --git a/src/main/java/com/odysseusinc/arachne/portal/model/DataSource.java b/src/main/java/com/odysseusinc/arachne/portal/model/DataSource.java index 717d20787..212b8d192 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/model/DataSource.java +++ b/src/main/java/com/odysseusinc/arachne/portal/model/DataSource.java @@ -22,23 +22,15 @@ package com.odysseusinc.arachne.portal.model; -import com.odysseusinc.arachne.commons.api.v1.dto.CommonCDMVersionDTO; -import com.odysseusinc.arachne.commons.api.v1.dto.CommonHealthStatus; -import com.odysseusinc.arachne.commons.api.v1.dto.CommonModelType; -import com.odysseusinc.arachne.portal.model.security.Tenant; -import com.odysseusinc.arachne.portal.model.solr.SolrFieldAnno; -import com.odysseusinc.arachne.portal.security.ArachnePermission; import com.odysseusinc.arachne.portal.security.HasArachnePermissions; -import java.io.Serializable; -import java.util.Date; -import java.util.HashSet; -import java.util.Set; -import javax.persistence.*; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; import org.hibernate.annotations.DiscriminatorFormula; import org.hibernate.annotations.SQLDelete; -import org.hibernate.validator.constraints.NotBlank; + +import javax.persistence.Entity; +import javax.persistence.Inheritance; +import javax.persistence.InheritanceType; +import javax.persistence.Table; +import java.io.Serializable; @Entity @Inheritance(strategy = InheritanceType.SINGLE_TABLE) diff --git a/src/main/java/com/odysseusinc/arachne/portal/model/IDataSource.java b/src/main/java/com/odysseusinc/arachne/portal/model/IDataSource.java index 8c345417a..97ed9baf6 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/model/IDataSource.java +++ b/src/main/java/com/odysseusinc/arachne/portal/model/IDataSource.java @@ -22,9 +22,9 @@ package com.odysseusinc.arachne.portal.model; -import com.odysseusinc.arachne.commons.api.v1.dto.CommonCDMVersionDTO; import com.odysseusinc.arachne.commons.api.v1.dto.CommonHealthStatus; import com.odysseusinc.arachne.commons.api.v1.dto.CommonModelType; +import com.odysseusinc.arachne.commons.types.CommonCDMVersionDTO; import com.odysseusinc.arachne.commons.types.DBMSType; import com.odysseusinc.arachne.portal.model.security.Tenant; import com.odysseusinc.arachne.portal.model.solr.SolrEntity; diff --git a/src/main/java/com/odysseusinc/arachne/portal/model/Submission.java b/src/main/java/com/odysseusinc/arachne/portal/model/Submission.java index 7eb515fe6..d9cd121f1 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/model/Submission.java +++ b/src/main/java/com/odysseusinc/arachne/portal/model/Submission.java @@ -23,6 +23,7 @@ package com.odysseusinc.arachne.portal.model; import com.google.common.base.Objects; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.odysseusinc.arachne.portal.api.v1.dto.InvitationType; import com.odysseusinc.arachne.portal.security.ArachnePermission; @@ -109,7 +110,7 @@ public class Submission implements HasArachnePermissions, Breadcrumb, Invitation @Column @Type(type = "com.odysseusinc.arachne.portal.repository.hibernate.JsonbType") - private JsonObject resultInfo; + private JsonElement resultInfo; @Column private Boolean hidden = false; @@ -334,12 +335,12 @@ public void setToken(String token) { this.token = token; } - public JsonObject getResultInfo() { + public JsonElement getResultInfo() { return resultInfo; } - public void setResultInfo(JsonObject resultInfo) { + public void setResultInfo(JsonElement resultInfo) { this.resultInfo = resultInfo; } diff --git a/src/main/java/com/odysseusinc/arachne/portal/repository/BaseAnalysisRepository.java b/src/main/java/com/odysseusinc/arachne/portal/repository/BaseAnalysisRepository.java index 3221c20a0..263f1860b 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/repository/BaseAnalysisRepository.java +++ b/src/main/java/com/odysseusinc/arachne/portal/repository/BaseAnalysisRepository.java @@ -24,6 +24,7 @@ import com.cosium.spring.data.jpa.entity.graph.domain.EntityGraph; import com.cosium.spring.data.jpa.entity.graph.repository.EntityGraphJpaRepository; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import com.odysseusinc.arachne.portal.model.Analysis; import com.odysseusinc.arachne.portal.model.Study; import java.util.List; @@ -54,4 +55,6 @@ public interface BaseAnalysisRepository extends EntityGraphJ List findByIdIn(List ids); List findByStudyIdOrderByOrd(Long studyId, EntityGraph entityGraph); + + List findByType(CommonAnalysisType type, EntityGraph entityGraph); } diff --git a/src/main/java/com/odysseusinc/arachne/portal/repository/BaseDataSourceRepository.java b/src/main/java/com/odysseusinc/arachne/portal/repository/BaseDataSourceRepository.java index f864de9d3..74fb23a97 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/repository/BaseDataSourceRepository.java +++ b/src/main/java/com/odysseusinc/arachne/portal/repository/BaseDataSourceRepository.java @@ -94,4 +94,6 @@ public interface BaseDataSourceRepository extends EntityG + " SET deleted_at = NULL " + " WHERE data_source_id = :dataSourceId") void makeLinksWithTenantsNotDeleted(@Param("dataSourceId") Long dataSourceId); + + List findByNameAndIdNot(String name, Long id); } diff --git a/src/main/java/com/odysseusinc/arachne/portal/repository/BaseStudyRepository.java b/src/main/java/com/odysseusinc/arachne/portal/repository/BaseStudyRepository.java index 2039f601c..d17c5f436 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/repository/BaseStudyRepository.java +++ b/src/main/java/com/odysseusinc/arachne/portal/repository/BaseStudyRepository.java @@ -29,7 +29,6 @@ import java.util.Collection; import java.util.List; import java.util.Optional; -import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.NoRepositoryBean; @@ -48,11 +47,13 @@ public interface BaseStudyRepository extends EntityGraphJpaRepo + " studies.id NOT IN (SELECT study_id FROM users_studies_extended " + " WHERE user_id=:participantId " + " AND lower(status) IN ('pending', 'approved')) " + + " AND studies.kind=:studyKind " + " AND studies_users.user_id=:ownerId AND lower(title) SIMILAR TO :suggestRequest" + " AND lower(studies_users.status) = 'approved'") - Iterable suggestByParticipantId(@Param("suggestRequest") String suggestRequest, + Iterable suggestByParticipantIdAndStudyKind(@Param("suggestRequest") String suggestRequest, @Param("ownerId") Long id, - @Param("participantId") Long participantId); + @Param("participantId") Long participantId, + @Param("studyKind") String studyKind); @Query(nativeQuery = true, value = "SELECT studies.* " + diff --git a/src/main/java/com/odysseusinc/arachne/portal/repository/DataNodeUserRepository.java b/src/main/java/com/odysseusinc/arachne/portal/repository/DataNodeUserRepository.java index f446d83b3..fb3b2fc9c 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/repository/DataNodeUserRepository.java +++ b/src/main/java/com/odysseusinc/arachne/portal/repository/DataNodeUserRepository.java @@ -33,4 +33,6 @@ public interface DataNodeUserRepository extends JpaRepository findByDataNode(DataNode dataNode); Optional findByDataNodeAndUserId(DataNode dataNode, Long userId); + + Optional findByDataNodeAndUser_Username(DataNode dataNode, String username); } diff --git a/src/main/java/com/odysseusinc/arachne/portal/repository/hibernate/JsonbType.java b/src/main/java/com/odysseusinc/arachne/portal/repository/hibernate/JsonbType.java index aedccb061..5766c03c5 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/repository/hibernate/JsonbType.java +++ b/src/main/java/com/odysseusinc/arachne/portal/repository/hibernate/JsonbType.java @@ -79,7 +79,7 @@ public Object nullSafeGet(ResultSet resultSet, String[] strings, SharedSessionCo } final JsonParser jsonParser = new JsonParser(); - return jsonParser.parse(json).getAsJsonObject(); + return jsonParser.parse(json); } @Override @@ -100,7 +100,7 @@ public Object deepCopy(Object value) throws HibernateException { return null; } final JsonParser jsonParser = new JsonParser(); - return jsonParser.parse(value.toString()).getAsJsonObject(); + return jsonParser.parse(value.toString()); } @Override diff --git a/src/main/java/com/odysseusinc/arachne/portal/security/AuthenticationTokenFilter.java b/src/main/java/com/odysseusinc/arachne/portal/security/AuthenticationTokenFilter.java index e1fb2ef1e..ebeaeb798 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/security/AuthenticationTokenFilter.java +++ b/src/main/java/com/odysseusinc/arachne/portal/security/AuthenticationTokenFilter.java @@ -22,8 +22,6 @@ package com.odysseusinc.arachne.portal.security; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.odysseusinc.arachne.commons.api.v1.dto.util.JsonResult; import com.odysseusinc.arachne.portal.config.tenancy.TenantContext; import com.odysseusinc.arachne.portal.model.security.ArachneUser; import java.io.IOException; @@ -34,38 +32,40 @@ import javax.servlet.ServletResponse; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import org.ohdsi.authenticator.exception.AuthenticationException; +import org.ohdsi.authenticator.service.Authenticator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpMethod; import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; -import org.springframework.security.core.AuthenticationException; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.core.userdetails.UserDetailsService; -import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.authentication.WebAuthenticationDetailsSource; +import org.springframework.web.filter.GenericFilterBean; -public class AuthenticationTokenFilter extends UsernamePasswordAuthenticationFilter { +public class AuthenticationTokenFilter extends GenericFilterBean { + + Logger log = LoggerFactory.getLogger(AuthenticationTokenFilter.class); public static final String USER_REQUEST_HEADER = "Arachne-User-Request"; @Value("${arachne.token.header}") private String tokenHeader; - @Autowired - private TokenUtils tokenUtils; - - @Autowired private UserDetailsService userDetailsService; - private ObjectMapper objectMapper = new ObjectMapper(); + @Autowired + private Authenticator authenticator; @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException, AuthenticationException { + HttpServletRequest httpRequest = (HttpServletRequest) request; try { - HttpServletRequest httpRequest = (HttpServletRequest) request; String authToken = httpRequest.getHeader(tokenHeader); if (authToken == null && httpRequest.getCookies() != null) { for (Cookie cookie : httpRequest.getCookies()) { @@ -75,41 +75,32 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha } } if (authToken != null) { - String username = this.tokenUtils.getUsernameFromToken(authToken); + String username = authenticator.resolveUsername(authToken); String requested = httpRequest.getHeader(USER_REQUEST_HEADER); - if (requested != null && username != null && !Objects.equals(username, requested)){ + if (requested != null && username != null && !Objects.equals(username, requested)) { throw new BadCredentialsException("forced logout"); } - if (tokenUtils.isExpired(authToken)) { - if (((HttpServletRequest) request).getRequestURI().startsWith("/api")) { - if (username != null) { - throw new BadCredentialsException("token expired"); - } - } - } - if (username != null && SecurityContextHolder.getContext().getAuthentication() == null) { UserDetails userDetails = this.userDetailsService.loadUserByUsername(username); - if (this.tokenUtils.validateToken(authToken, userDetails)) { - UsernamePasswordAuthenticationToken authentication = - new UsernamePasswordAuthenticationToken(userDetails, null, userDetails.getAuthorities()); - authentication.setDetails(new WebAuthenticationDetailsSource().buildDetails(httpRequest)); - SecurityContextHolder.getContext().setAuthentication(authentication); + UsernamePasswordAuthenticationToken authentication = + new UsernamePasswordAuthenticationToken(userDetails, null, userDetails.getAuthorities()); + authentication.setDetails(new WebAuthenticationDetailsSource().buildDetails(httpRequest)); + SecurityContextHolder.getContext().setAuthentication(authentication); - TenantContext.setCurrentTenant(((ArachneUser) userDetails).getActiveTenantId()); - } + TenantContext.setCurrentTenant(((ArachneUser) userDetails).getActiveTenantId()); + } + } + } catch (AuthenticationException | org.springframework.security.core.AuthenticationException ex) { + String method = httpRequest.getMethod(); + if (!HttpMethod.OPTIONS.matches(method)) { + if (log.isDebugEnabled()) { + log.debug("Authentication failed", ex); + } else { + log.error("Authentication failed: {}, requested: {} {}", ex.getMessage(), method, httpRequest.getRequestURI()); } } - chain.doFilter(request, response); - } catch (AuthenticationException ex) { - logger.debug(ex.getMessage(), ex); - ((HttpServletResponse) response).setStatus(HttpServletResponse.SC_UNAUTHORIZED); - JsonResult result = new JsonResult<>(JsonResult.ErrorCode.UNAUTHORIZED); - result.setResult(Boolean.FALSE); - - response.getOutputStream().write(objectMapper.writeValueAsString(result).getBytes()); - response.setContentType("application/json"); } + chain.doFilter(request, response); } } diff --git a/src/main/java/com/odysseusinc/arachne/portal/security/TokenUtils.java b/src/main/java/com/odysseusinc/arachne/portal/security/TokenUtils.java deleted file mode 100644 index 7b711aa22..000000000 --- a/src/main/java/com/odysseusinc/arachne/portal/security/TokenUtils.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * - * Copyright 2018 Odysseus Data Services, inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Company: Odysseus Data Services, Inc. - * Product Owner/Architecture: Gregory Klebanov - * Authors: Pavel Grafkin, Alexandr Ryabokon, Vitaly Koulakov, Anton Gackovka, Maria Pozhidaeva, Mikhail Mironov - * Created: October 19, 2016 - * - */ - -package com.odysseusinc.arachne.portal.security; - -import com.odysseusinc.arachne.portal.model.security.ArachneUser; -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Jwts; -import io.jsonwebtoken.SignatureAlgorithm; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import javax.servlet.http.HttpServletRequest; -import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.security.core.userdetails.UserDetails; -import org.springframework.stereotype.Component; - -@Component -public class TokenUtils { - - public static final String EX_CONCURRENT_LOGIN = "User %s token invalidated due to concurrent login"; - private final Logger log = Logger.getLogger(getClass()); - private final Object monitor = new Object(); - @Value("${arachne.token.header}") - private String tokenHeader; - @Value("${arachne.token.secret}") - private String secret; - @Value("${arachne.token.expiration}") - private Long expiration; - private ConcurrentHashMap invalidatedTokens = new ConcurrentHashMap<>(); - - private ConcurrentHashMap usernameTokenMap = new ConcurrentHashMap<>(); - - public List getAuthToken(HttpServletRequest request) { - - List tokens = new ArrayList<>(); - - // Get token from header - String headerToken = request.getHeader(tokenHeader); - if (headerToken != null) { - tokens.add(headerToken); - } - - // Get token from cookie - if (request.getCookies() != null) { - Arrays.stream(request.getCookies()) - .filter(cookie -> cookie.getName().equalsIgnoreCase(tokenHeader) && cookie.getValue() != null) - .findFirst() - .ifPresent(cookie -> tokens.add(cookie.getValue())); - } - - return tokens; - } - - public boolean isExpired(String token) { - - boolean expired; - try { - Claims claims = getClaimsFromToken(token); - expired = claims.getExpiration().getTime() * 1000 < new Date().getTime(); - } catch (Exception ex) { - log.debug(ex.getMessage(), ex); - expired = true; - } - return expired; - } - - - public String getUsernameFromToken(String token) { - - String username; - try { - final Claims claims = getClaimsFromToken(token); - username = claims.getSubject(); - } catch (Exception ex) { - log.debug(ex.getMessage(), ex); - username = null; - } - return username; - } - - public String getUUIDFromToken(String token) { - - String uuid; - try { - final Claims claims = getClaimsFromToken(token); - uuid = claims.get("uuid", String.class); - } catch (Exception ex) { - log.debug(ex.getMessage(), ex); - uuid = null; - } - return uuid; - } - - public Date getCreatedDateFromToken(String token) { - - Date created; - try { - final Claims claims = getClaimsFromToken(token); - created = new Date((Long) claims.get("created")); - } catch (Exception ex) { - log.debug(ex.getMessage(), ex); - created = null; - } - return created; - } - - public Date getExpirationDateFromToken(String token) { - - Date expiration; - try { - final Claims claims = getClaimsFromToken(token); - expiration = claims.getExpiration(); - } catch (Exception ex) { - log.debug(ex.getMessage(), ex); - expiration = null; - } - return expiration; - } - - - private Claims getClaimsFromToken(String token) { - - Claims claims; - try { - claims = Jwts.parser() - .setSigningKey(secret) - .parseClaimsJws(token) - .getBody(); - } catch (Exception ex) { - log.debug(ex.getMessage(), ex); - claims = null; - } - return claims; - } - - private Date generateCurrentDate() { - - return new Date(System.currentTimeMillis()); - } - - private Date generateExpirationDate() { - - return new Date(System.currentTimeMillis() + expiration * 1000); - } - - private Boolean isTokenExpired(String token) { - - final Date expiration = getExpirationDateFromToken(token); - return expiration.before(generateCurrentDate()); - } - - private Boolean isCreatedBeforeLastPasswordReset(Date created, Date lastPasswordReset) { - - return (lastPasswordReset != null && created.before(lastPasswordReset)); - } - - - public String generateToken(String username) { - - Map claims = new HashMap(); - claims.put("sub", username); - claims.put("created", generateCurrentDate()); - claims.put("uuid", UUID.randomUUID().toString()); - - return checkTokenConcurrency(username, generateToken(claims)); - } - - private String checkTokenConcurrency(final String username, final String token) { - - synchronized (monitor) { - String oldToken = usernameTokenMap.getOrDefault(username, null); - if (!Objects.equals(token, oldToken) && (Objects.nonNull(oldToken) && !isExpired(oldToken))) { - log.info(String.format(EX_CONCURRENT_LOGIN, username)); - addInvalidateToken(oldToken); - } - usernameTokenMap.put(username, token); - } - return token; - } - - private String generateToken(Map claims) { - - return Jwts.builder() - .setClaims(claims) - .setExpiration(generateExpirationDate()) - .signWith(SignatureAlgorithm.HS512, secret) - .compact(); - } - - public Boolean canTokenBeRefreshed(String token, Date lastPasswordReset) { - - final Date created = getCreatedDateFromToken(token); - String uuid = getUUIDFromToken(token); - return (!(invalidatedTokens.containsKey(uuid)) - && !(isCreatedBeforeLastPasswordReset(created, lastPasswordReset)) - && (!(isTokenExpired(token)))); - } - - public String refreshToken(String token) { - - String refreshedToken; - try { - final Claims claims = getClaimsFromToken(token); - claims.put("created", generateCurrentDate()); - refreshedToken = generateToken(claims); - } catch (Exception ex) { - log.debug(ex.getMessage(), ex); - refreshedToken = null; - } - return refreshedToken; - } - - public Boolean validateToken(String token, UserDetails userDetails) { - - String uuid = getUUIDFromToken(token); - boolean result = false; - if (!invalidatedTokens.containsKey(uuid)) { - ArachneUser user = (ArachneUser) userDetails; - final String username = getUsernameFromToken(token); - final Date created = getCreatedDateFromToken(token); - result = (username.equals(user.getUsername()) - && !(isTokenExpired(token)) - && !(isCreatedBeforeLastPasswordReset(created, user.getLastPasswordReset()))); - } - return result; - } - - public void addInvalidateToken(String token) { - - String uuid = getUUIDFromToken(token); - Date expirationDate = getExpirationDateFromToken(token); - Date now = new Date(); - if ((expirationDate == null || expirationDate.after(now)) && uuid != null) { - invalidatedTokens.put(uuid, expirationDate); - } - //remove old tokens - for (Iterator> iterator = invalidatedTokens.entrySet().iterator(); iterator.hasNext(); ) { - Map.Entry stringDateEntry = iterator.next(); - if (stringDateEntry.getValue() != null && now.after(stringDateEntry.getValue())) { - iterator.remove(); - } - } - } -} diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/LoginAttemptService.java b/src/main/java/com/odysseusinc/arachne/portal/service/LoginAttemptService.java index 0968a654a..9a18f1963 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/LoginAttemptService.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/LoginAttemptService.java @@ -29,5 +29,5 @@ public interface LoginAttemptService { void loginFailed(String key); - boolean isBlocked(String key); + Long getRemainingAccountLockPeriod(String key); } diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/analysis/BaseAnalysisService.java b/src/main/java/com/odysseusinc/arachne/portal/service/analysis/BaseAnalysisService.java index 944a4e568..707d95020 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/analysis/BaseAnalysisService.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/analysis/BaseAnalysisService.java @@ -127,6 +127,8 @@ void updateCodeFile(AnalysisFile analysisFile, List getByStudyId(Long id, EntityGraph author); + List findByType(CommonAnalysisType type); + void processAntivirusResponse(AntivirusJobAnalysisFileResponseEvent event); void indexAllBySolr() diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/analysis/impl/BaseAnalysisServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/analysis/impl/BaseAnalysisServiceImpl.java index 3fbd4bbda..6c2c854c3 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/analysis/impl/BaseAnalysisServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/analysis/impl/BaseAnalysisServiceImpl.java @@ -31,6 +31,7 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import com.cosium.spring.data.jpa.entity.graph.domain.EntityGraph; +import com.cosium.spring.data.jpa.entity.graph.domain.EntityGraphUtils; import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -1005,4 +1006,11 @@ public void indexBySolr(final A analysis) solrService.indexBySolr(analysis); } + + @Override + @Transactional + public List findByType(CommonAnalysisType type) { + + return analysisRepository.findByType(type, EntityGraphUtils.fromAttributePaths("submissions", "submissions.submissionGroup")); + } } diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/impl/AnalysisPreprocessorService.java b/src/main/java/com/odysseusinc/arachne/portal/service/impl/AnalysisPreprocessorService.java index deab9c219..6bca96451 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/impl/AnalysisPreprocessorService.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/impl/AnalysisPreprocessorService.java @@ -52,7 +52,7 @@ public AnalysisPreprocessorService(AnalysisHelper analysisHelper, public void preprocessFile(Analysis analysis, AnalysisFile file) { Path analysisFolder = analysisHelper.getAnalysisFolder(analysis); - File target = analysisFolder.resolve(file.getRealName()).toFile(); + File target = analysisFolder.resolve(file.getName()).toFile(); getPreprocessorRegistry().getPreprocessor(file.getContentType()) .preprocess(analysis, target); } @@ -64,7 +64,7 @@ protected List getFiles(Analysis analysis) { return analysis .getFiles() .stream() - .map(analysisFile -> analysisFolder.resolve(analysisFile.getRealName()).toFile()) + .map(analysisFile -> analysisFolder.resolve(analysisFile.getName()).toFile()) .collect(Collectors.toList()); } @@ -74,7 +74,7 @@ protected Optional getContentType(Analysis analysis, File file) { return analysis .getFiles() .stream() - .filter(analysisFile -> analysisFile.getRealName().equals(file.getName())) + .filter(analysisFile -> analysisFile.getName().equals(file.getName())) .findFirst().map(ArachneFile::getContentType); } } diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseArachneSecureServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseArachneSecureServiceImpl.java index abafd3ceb..08ea08dd2 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseArachneSecureServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseArachneSecureServiceImpl.java @@ -248,9 +248,9 @@ public boolean wasDataSourceApproved(Analysis analysis, Long dataSourceId) { public boolean checkDataNodeAdmin(ArachneUser user, DataNode dataNode) { final RawUser standardUser = new RawUser(); - standardUser.setId(user.getId()); + standardUser.setUsername(user.getUsername()); - return dataNodeUserRepository.findByDataNodeAndUserId(dataNode, standardUser.getId()).isPresent(); + return dataNodeUserRepository.findByDataNodeAndUser_Username(dataNode, standardUser.getUsername()).isPresent(); } @Override diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseDataNodeServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseDataNodeServiceImpl.java index 4151df3b0..436882bc8 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseDataNodeServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseDataNodeServiceImpl.java @@ -49,6 +49,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Predicate; import javax.validation.ConstraintViolationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -230,7 +231,7 @@ public void unlinkUserToDataNode(DN dataNode, IUser user) throws NotExistExcepti LOGGER.info(UNLINKING_USER_LOG, user.getId(), dataNode.getId()); final DataNodeUser existDataNodeUser - = dataNodeUserRepository.findByDataNodeAndUserId(dataNode, user.getId()) + = dataNodeUserRepository.findByDataNodeAndUser_Username(dataNode, user.getUsername()) .orElseThrow(() -> { final String message = String.format(USER_IS_NOT_LINKED_EXC, user.getId(), dataNode.getId()); @@ -239,6 +240,8 @@ public void unlinkUserToDataNode(DN dataNode, IUser user) throws NotExistExcepti dataNodeUserRepository.delete(existDataNodeUser); } + private Predicate filterNullUsers = u -> Objects.nonNull(u.getUser()); + @Transactional @Override @PreAuthorize("#dataNode == authentication.principal") @@ -246,12 +249,12 @@ public void relinkAllUsersToDataNode(DN dataNode, Set dataNodeUser LOGGER.info(RELINKING_ALL_USERS_LOG, dataNode.getId()); final List existingUsers = dataNodeUserRepository.findByDataNode(dataNode); - existingUsers.forEach(existingDataNodeUser -> { - if (!dataNodeUsers.contains(existingDataNodeUser)) { + existingUsers.stream().filter(filterNullUsers).forEach(existingDataNodeUser -> { + if (dataNodeUsers.stream().noneMatch(dnu -> Objects.equals(dnu.getUser().getUsername(), existingDataNodeUser.getUser().getUsername()))) { dataNodeUserRepository.delete(existingDataNodeUser); } }); - dataNodeUsers.stream().filter(user -> Objects.nonNull(user.getUser())).forEach(dataNodeUser -> { + dataNodeUsers.stream().filter(filterNullUsers).forEach(dataNodeUser -> { dataNodeUser.setDataNode(dataNode); saveOrUpdateDataNodeUser(dataNode, dataNodeUser); }); @@ -273,7 +276,7 @@ public Optional findByToken(String token) { private void saveOrUpdateDataNodeUser(DataNode dataNode, DataNodeUser dataNodeUser) { dataNodeUser.setDataNode(dataNode); - dataNodeUserRepository.findByDataNodeAndUserId(dataNode, dataNodeUser.getUser().getId()) + dataNodeUserRepository.findByDataNodeAndUser_Username(dataNode, dataNodeUser.getUser().getUsername()) .ifPresent(existing -> dataNodeUser.setId(existing.getId())); dataNodeUserRepository.save(dataNodeUser); } diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseDataSourceServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseDataSourceServiceImpl.java index e1fbdaec7..818d7d518 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseDataSourceServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseDataSourceServiceImpl.java @@ -29,6 +29,7 @@ import com.odysseusinc.arachne.portal.config.tenancy.TenantContext; import com.odysseusinc.arachne.portal.exception.FieldException; import com.odysseusinc.arachne.portal.exception.NotExistException; +import com.odysseusinc.arachne.portal.exception.NotUniqueException; import com.odysseusinc.arachne.portal.exception.PermissionDeniedException; import com.odysseusinc.arachne.portal.model.DataSource; import com.odysseusinc.arachne.portal.model.DataSourceSortMapper; @@ -214,6 +215,9 @@ public DS updateInAnyTenant(DS dataSource, Consumer> beforeU throws IllegalAccessException, NoSuchFieldException, SolrServerException, IOException { DS forUpdate = getByIdInAnyTenant(dataSource.getId()); + if (!isNameUnique(forUpdate.getId(), dataSource.getName())) { + throw new NotUniqueException("name", "Data source name is not unique"); + } forUpdate = baseUpdate(forUpdate, dataSource); beforeUpdate.accept(new PairForUpdating<>(forUpdate, dataSource)); @@ -245,9 +249,14 @@ public DS updateWithoutMetadataInAnyTenant(DS dataSource) }); } + private boolean isNameUnique(Long sourceId, String dsName) { + + return dataSourceRepository.findByNameAndIdNot(dsName, sourceId).isEmpty(); + } + private DS baseUpdate(DS exist, DS dataSource) { - if (dataSource.getName() != null) { + if (StringUtils.isNotBlank(dataSource.getName())) { exist.setName(dataSource.getName()); } diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseStudyServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseStudyServiceImpl.java index ec584082f..c43d1259e 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseStudyServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/impl/BaseStudyServiceImpl.java @@ -961,7 +961,7 @@ public Iterable suggestStudy(String query, IUser owner, Long id, SuggestSearc final String suggestRequest = "%" + query.toLowerCase() + "%"; switch (region) { case PARTICIPANT: { - suggest = studyRepository.suggestByParticipantId(suggestRequest, owner.getId(), id); + suggest = studyRepository.suggestByParticipantIdAndStudyKind(suggestRequest, owner.getId(), id, StudyKind.REGULAR.name()); break; } diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/impl/ImportServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/impl/ImportServiceImpl.java index 19f192ea2..1ce5961f5 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/impl/ImportServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/impl/ImportServiceImpl.java @@ -22,13 +22,9 @@ package com.odysseusinc.arachne.portal.service.impl; -import static com.odysseusinc.arachne.commons.utils.TemplateUtils.loadTemplate; - -import com.github.jknack.handlebars.Template; import com.odysseusinc.arachne.portal.service.ImportService; import com.odysseusinc.arachne.portal.util.ImportedFile; import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/impl/LoginAttemptServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/impl/LoginAttemptServiceImpl.java index 54b347e4e..967faa1cf 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/impl/LoginAttemptServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/impl/LoginAttemptServiceImpl.java @@ -22,17 +22,18 @@ package com.odysseusinc.arachne.portal.service.impl; +import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import com.odysseusinc.arachne.portal.service.LoginAttemptService; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import javax.annotation.PostConstruct; -import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.concurrent.TimeUnit; + @Service public class LoginAttemptServiceImpl implements LoginAttemptService { @@ -42,38 +43,40 @@ public class LoginAttemptServiceImpl implements LoginAttemptService { @Value("${arachne.loginAttempts.resetMinutes}") private int attemptsResetMinutes; - private LoadingCache attemptsCache; + private Cache> attemptsCache; @PostConstruct private void init() { attemptsCache = CacheBuilder.newBuilder(). - expireAfterWrite(attemptsResetMinutes, TimeUnit.MINUTES).build(new CacheLoader() { - public Integer load(String key) { - - return 0; - } - }); + expireAfterWrite(attemptsResetMinutes, TimeUnit.MINUTES).build(); } + @Override public void loginSucceeded(String key) { attemptsCache.invalidate(key); } + @Override public void loginFailed(String key) { - int attempts = ObjectUtils.firstNonNull(attemptsCache.getIfPresent(key), 0); - attempts++; - attemptsCache.put(key, attempts); + final Pair attemptsCounter = attemptsCache.getIfPresent(key); + if (attemptsCounter == null) { + attemptsCache.put(key, Pair.of(1, LocalDateTime.now())); + } else if (attemptsCounter.getKey() <= maxAttempts) { + final Integer failedAttemptsBefore = attemptsCounter.getKey(); + attemptsCache.put(key, Pair.of(failedAttemptsBefore + 1, LocalDateTime.now())); + } } - public boolean isBlocked(String key) { + @Override + public Long getRemainingAccountLockPeriod(String key) { - try { - return attemptsCache.get(key) >= maxAttempts; - } catch (ExecutionException e) { - return false; + final Pair attemptsCounter = this.attemptsCache.getIfPresent(key); + if (attemptsCounter != null && attemptsCounter.getKey() >= maxAttempts) { + return Duration.between(LocalDateTime.now(), attemptsCounter.getValue().plusMinutes(attemptsResetMinutes)).getSeconds(); } + return null; } } diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/impl/UserDetailsServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/impl/UserDetailsServiceImpl.java index 9da7f1fb5..bc9031112 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/impl/UserDetailsServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/impl/UserDetailsServiceImpl.java @@ -44,15 +44,15 @@ public class UserDetailsServiceImpl implements UserDetailsService { private UserService userService; @Override - public UserDetails loadUserByUsername(String email) throws UsernameNotFoundException { + public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException { - IUser user = userService.getByEmailInAnyTenant(email); + IUser user = userService.getByUsernameInAnyTenant(username); if (user == null) { - throw new UsernameNotFoundException(String.format("No user found with email '%s'.", email)); + throw new UsernameNotFoundException(String.format("No user found with username '%s'.", username)); } if (!user.getEnabled()) { if (!user.getEmailConfirmed()) { - throw new UserNotActivatedException(String.format("User with email='%s' is not activated", email)); + throw new UserNotActivatedException(String.format("User with username='%s' is not activated", username)); } throw new BadCredentialsException(ErrorMessages.BAD_CREDENTIALS.getMessage()); // temp solution - change in ARACHNE-2490 } diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/submission/BaseSubmissionService.java b/src/main/java/com/odysseusinc/arachne/portal/service/submission/BaseSubmissionService.java index 928f368cd..da1c45ab6 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/submission/BaseSubmissionService.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/submission/BaseSubmissionService.java @@ -41,15 +41,15 @@ import com.odysseusinc.arachne.portal.service.impl.submission.SubmissionAction; import com.odysseusinc.arachne.storage.model.ArachneFileMeta; import com.odysseusinc.arachne.storage.util.FileSaveRequest; -import org.springframework.data.domain.Page; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.multipart.MultipartFile; - +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Path; import java.util.List; +import org.springframework.data.domain.Page; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.multipart.MultipartFile; public interface BaseSubmissionService { T approveSubmissionResult(Long submissionId, ApproveDTO approveDTO, IUser user); @@ -99,7 +99,9 @@ T getSubmissionByIdAndUpdatePasswordAndStatus(Long id, String updatePassword, Li void notifyOwnersAboutSubmissionUpdateViaSocket(T submission); - ResultFile uploadResultsByDataOwner(Long submissionId, String name, MultipartFile file) throws NotExistException, IOException; + void uploadResultsByDataOwner(Long submissionId, File compressedFile) throws IOException; + + ResultFile uploadResultsByDataOwner(Long submissionId, String fileName, File file) throws IOException; @PreAuthorize("hasPermission(#submissionId, 'Submission', " + "T(com.odysseusinc.arachne.portal.security.ArachnePermission).ACCESS_STUDY)") diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/submission/impl/BaseSubmissionServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/submission/impl/BaseSubmissionServiceImpl.java index 9d64b9cfe..877ef772d 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/submission/impl/BaseSubmissionServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/submission/impl/BaseSubmissionServiceImpl.java @@ -31,13 +31,12 @@ import static com.odysseusinc.arachne.portal.model.SubmissionStatus.IN_PROGRESS; import static com.odysseusinc.arachne.portal.model.SubmissionStatus.NOT_APPROVED; import static com.odysseusinc.arachne.portal.model.SubmissionStatus.PENDING; -import static com.odysseusinc.arachne.portal.model.SubmissionStatus.valueOf; import static com.odysseusinc.arachne.portal.service.impl.submission.SubmissionActionType.EXECUTE; import static com.odysseusinc.arachne.portal.service.impl.submission.SubmissionActionType.PUBLISH; import static com.odysseusinc.arachne.portal.util.DataNodeUtils.isDataNodeOwner; import com.cosium.spring.data.jpa.entity.graph.domain.EntityGraph; -import com.cosium.spring.data.jpa.entity.graph.domain.EntityGraphUtils; +import com.odysseusinc.arachne.commons.utils.UUIDGenerator; import com.odysseusinc.arachne.portal.api.v1.dto.ApproveDTO; import com.odysseusinc.arachne.portal.api.v1.dto.UpdateNotificationDTO; import com.odysseusinc.arachne.portal.config.WebSecurityConfig; @@ -79,7 +78,6 @@ import com.odysseusinc.arachne.portal.util.EntityUtils; import com.odysseusinc.arachne.portal.util.LegacyAnalysisHelper; import com.odysseusinc.arachne.portal.util.SubmissionHelper; -import com.odysseusinc.arachne.portal.util.UUIDGenerator; import com.odysseusinc.arachne.portal.util.ZipUtil; import com.odysseusinc.arachne.storage.model.ArachneFileMeta; import com.odysseusinc.arachne.storage.model.QuerySpec; @@ -111,6 +109,9 @@ import java.util.zip.ZipOutputStream; import javax.persistence.EntityManager; import javax.transaction.Transactional; +import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.model.FileHeader; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,6 +126,7 @@ import org.springframework.security.core.userdetails.UserDetails; import org.springframework.util.CollectionUtils; import org.springframework.util.DigestUtils; +import org.springframework.util.StringUtils; import org.springframework.web.multipart.MultipartFile; public abstract class BaseSubmissionServiceImpl< @@ -132,13 +134,14 @@ public abstract class BaseSubmissionServiceImpl< A extends Analysis, DS extends IDataSource> implements BaseSubmissionService { + protected static final Logger LOGGER = LoggerFactory.getLogger(BaseSubmissionService.class); public static final String SUBMISSION_NOT_EXIST_EXCEPTION = "Submission with id='%s' does not exist"; public static final String ILLEGAL_SUBMISSION_STATE_EXCEPTION = "Submission must be in EXECUTED or FAILED state before approve result"; public static final String RESULT_FILE_NOT_EXISTS_EXCEPTION = "Result file with id='%s' for submission with " + "id='%s' does not exist"; - protected static final Logger LOGGER = LoggerFactory.getLogger(BaseSubmissionService.class); private static final String FILE_NOT_UPLOADED_MANUALLY_EXCEPTION = "File %s was not uploaded manually"; + protected final BaseSubmissionRepository submissionRepository; protected final BaseDataSourceService dataSourceService; protected final ArachneMailSender mailSender; @@ -533,8 +536,27 @@ public T getSubmissionByIdAndToken(Long id, String token) throws NotExistExcepti return submissionRepository.findByIdAndToken(id, token).orElseThrow(() -> new NotExistException(Submission.class)); } + + @Override + public void uploadResultsByDataOwner(Long submissionId, File compressedFile) throws IOException { + ZipFile zipFile = new ZipFile(compressedFile); + final List fileHeaders = zipFile.getFileHeaders().stream() + .filter(fileHeader -> !fileHeader.isDirectory()) + .collect(Collectors.toList()); + + for (FileHeader fileHeader : fileHeaders) { + File tempFile = File.createTempFile(fileHeader.getFileName(), ""); + tempFile.deleteOnExit(); + FileUtils.copyInputStreamToFile(zipFile.getInputStream(fileHeader), tempFile); + + String filename = StringUtils.getFilename(fileHeader.getFileName()); + uploadResultsByDataOwner(submissionId, filename, tempFile); + } + } + + @Override - public ResultFile uploadResultsByDataOwner(Long submissionId, String name, MultipartFile file) throws NotExistException, IOException { + public ResultFile uploadResultsByDataOwner(Long submissionId, String fileName, File file) throws IOException { UserDetails userDetails = (UserDetails) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); IUser user = userService.getByUsername(userDetails.getUsername()); @@ -543,12 +565,9 @@ public ResultFile uploadResultsByDataOwner(Long submissionId, String name, Multi Collections.singletonList(IN_PROGRESS.name())); throwNotExistExceptionIfNull(submission, submissionId); - File tempFile = File.createTempFile("manual-upload", name); - file.transferTo(tempFile); - ResultFile resultFile = createResultFile( - tempFile.toPath(), - ObjectUtils.firstNonNull(name, file.getOriginalFilename()), + file.toPath(), + fileName, submission, user.getId() ); diff --git a/src/main/java/com/odysseusinc/arachne/portal/service/submission/impl/SubmissionServiceImpl.java b/src/main/java/com/odysseusinc/arachne/portal/service/submission/impl/SubmissionServiceImpl.java index e1be540ef..e7fd99ab6 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/service/submission/impl/SubmissionServiceImpl.java +++ b/src/main/java/com/odysseusinc/arachne/portal/service/submission/impl/SubmissionServiceImpl.java @@ -53,10 +53,12 @@ import com.odysseusinc.arachne.portal.util.SubmissionHelper; import com.odysseusinc.arachne.storage.model.ArachneFileMeta; import com.odysseusinc.arachne.storage.service.ContentStorageService; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.util.List; import javax.persistence.EntityManager; +import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.simp.SimpMessagingTemplate; import org.springframework.security.access.prepost.PostAuthorize; @@ -147,11 +149,18 @@ public boolean deleteSubmissionResultFile(Long submissionId, ResultFile resultFi return super.deleteSubmissionResultFile(submissionId, resultFile); } + @Override @PreAuthorize("hasPermission(#submissionId, 'Submission', " + "T(com.odysseusinc.arachne.portal.security.ArachnePermission).APPROVE_SUBMISSION)") - public ResultFile uploadResultsByDataOwner(Long submissionId, String name, MultipartFile file) throws NotExistException, IOException { + public void uploadResultsByDataOwner(Long submissionId, File compressedFile) throws IOException { + super.uploadResultsByDataOwner(submissionId, compressedFile); + } + @Override + @PreAuthorize("hasPermission(#submissionId, 'Submission', " + + "T(com.odysseusinc.arachne.portal.security.ArachnePermission).APPROVE_SUBMISSION)") + public ResultFile uploadResultsByDataOwner(Long submissionId, String name, File file) throws NotExistException, IOException { return super.uploadResultsByDataOwner(submissionId, name, file); } diff --git a/src/main/java/com/odysseusinc/arachne/portal/util/AnalysisHelper.java b/src/main/java/com/odysseusinc/arachne/portal/util/AnalysisHelper.java index af5814427..3a6a56f1c 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/util/AnalysisHelper.java +++ b/src/main/java/com/odysseusinc/arachne/portal/util/AnalysisHelper.java @@ -44,15 +44,17 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; -import net.lingala.zip4j.core.ZipFile; +import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.model.ZipParameters; -import net.lingala.zip4j.util.Zip4jConstants; +import net.lingala.zip4j.model.enums.CompressionLevel; +import net.lingala.zip4j.model.enums.CompressionMethod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; + @Component public class AnalysisHelper implements AnalysisPaths { @@ -156,9 +158,9 @@ public void compressAndSplit(ArrayList files, File zipArchive) { try { ZipFile zipFile = new ZipFile(zipArchive.getAbsoluteFile()); ZipParameters parameters = new ZipParameters(); - parameters.setCompressionMethod(Zip4jConstants.COMP_DEFLATE); - parameters.setCompressionLevel(Zip4jConstants.DEFLATE_LEVEL_NORMAL); - zipFile.createZipFile(files, parameters, true, maximumSize); + parameters.setCompressionMethod(CompressionMethod.DEFLATE); + parameters.setCompressionLevel(CompressionLevel.NORMAL); + zipFile.createSplitZipFile(files, parameters, true, maximumSize); } catch (ZipException ex) { LOGGER.error(ex.getMessage(), ex); throw new ConverterRuntimeException(ex.getMessage()); diff --git a/src/main/java/com/odysseusinc/arachne/portal/util/BaseStudyHelper.java b/src/main/java/com/odysseusinc/arachne/portal/util/BaseStudyHelper.java index c73fe4a14..d0e05d8b4 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/util/BaseStudyHelper.java +++ b/src/main/java/com/odysseusinc/arachne/portal/util/BaseStudyHelper.java @@ -25,8 +25,8 @@ import static com.odysseusinc.arachne.commons.api.v1.dto.CommonModelType.CDM; import static com.odysseusinc.arachne.portal.service.AnalysisPaths.CONTENT_DIR; -import com.odysseusinc.arachne.commons.api.v1.dto.CommonCDMVersionDTO; import com.odysseusinc.arachne.commons.api.v1.dto.CommonHealthStatus; +import com.odysseusinc.arachne.commons.types.CommonCDMVersionDTO; import com.odysseusinc.arachne.portal.model.DataNode; import com.odysseusinc.arachne.portal.model.DataNodeUser; import com.odysseusinc.arachne.portal.model.IDataSource; diff --git a/src/main/java/com/odysseusinc/arachne/portal/util/SubmissionHelper.java b/src/main/java/com/odysseusinc/arachne/portal/util/SubmissionHelper.java index 0ec8d17a5..3860d5b90 100644 --- a/src/main/java/com/odysseusinc/arachne/portal/util/SubmissionHelper.java +++ b/src/main/java/com/odysseusinc/arachne/portal/util/SubmissionHelper.java @@ -26,30 +26,48 @@ import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import com.odysseusinc.arachne.commons.utils.cohortcharacterization.CohortCharacterizationDocType; +import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import com.odysseusinc.arachne.portal.model.Submission; import com.odysseusinc.arachne.portal.repository.SubmissionResultFileRepository; import com.odysseusinc.arachne.storage.model.ArachneFileMeta; import com.odysseusinc.arachne.storage.model.QuerySpec; import com.odysseusinc.arachne.storage.service.ContentStorageService; import com.odysseusinc.arachne.storage.service.JcrContentStorageServiceImpl; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; @Component @@ -75,6 +93,7 @@ public SubmissionHelper(SubmissionResultFileRepository submissionResultFileRepos this.contentStorageHelper = contentStorageHelper; } + @Transactional public void updateSubmissionExtendedInfo(final Submission submission) { final CommonAnalysisType analysisType = submission.getSubmissionGroup().getAnalysisType(); @@ -204,56 +223,116 @@ public void updateExtendInfo(final Submission submission) { private class IncidenceSubmissionExtendInfoStrategy extends SubmissionExtendInfoAnalyzeStrategy { + private static final String STUDY_SPECIFICATION_JSON = "StudySpecification.json"; + @Override public void updateExtendInfo(Submission submission) { - final JsonObject resultInfo = new JsonObject(); + final JsonArray result = new JsonArray(); try { final String resultsDir = contentStorageHelper.getResultFilesDir(submission); + final Map cohortNames = new HashMap<>(); + List packageFiles = searchFiles(submission, "IncidenceRate%.zip"); + if (!packageFiles.isEmpty()) { + Path tmpDir = Files.createTempDirectory("incidencerate"); + try { + File archiveFile = tmpDir.resolve("IncidenceRate.zip").toFile(); + try(OutputStream out = new FileOutputStream(archiveFile); + InputStream fileIn = contentStorageService.getContentByFilepath(packageFiles.get(0).getPath())) { + IOUtils.copy(fileIn, out); + } + try(FileInputStream in = new FileInputStream(archiveFile); + ZipInputStream zip = new ZipInputStream(in)) { + ZipEntry entry = zip.getNextEntry(); + while(entry != null) { + if (entry.getName().endsWith(STUDY_SPECIFICATION_JSON)) { + File jsonFile = tmpDir.resolve(STUDY_SPECIFICATION_JSON).toFile(); + try(FileOutputStream out = new FileOutputStream(jsonFile)) { + IOUtils.copy(zip, out); + } + } + zip.closeEntry(); + entry = zip.getNextEntry(); + } + } + + Path specFile = tmpDir.resolve(STUDY_SPECIFICATION_JSON); + if (Files.exists(specFile) && Files.isRegularFile(specFile)) { + JsonParser parser = new JsonParser(); + try (Reader json = new FileReader(specFile.toFile())) { + JsonObject spec = parser.parse(json).getAsJsonObject(); + JsonArray targets = spec.get("targetCohorts").getAsJsonArray(); + cohortNames.putAll(getCohortNames(targets)); + JsonArray outcomes = spec.get("outcomeCohorts").getAsJsonArray(); + cohortNames.putAll(getCohortNames(outcomes)); + } + } + } catch (Exception e) { + LOGGER.warn("Failed to parse cohort names, {}", e.getMessage()); + } finally { + FileUtils.deleteQuietly(tmpDir.toFile()); + } + } + ArachneFileMeta arachneFile = contentStorageService.getFileByPath(resultsDir + JcrContentStorageServiceImpl.PATH_SEPARATOR + INCIDENCE_SUMMARY_FILENAME); final CSVParser parser = CSVParser.parse(contentStorageService.getContentByFilepath(arachneFile.getPath()), Charset.defaultCharset(), CSVFormat.DEFAULT.withHeader()); final Map headers = parser.getHeaderMap(); + final String targetIdHeader = "TARGET_ID"; + final String outcomeIdHeader = "OUTCOME_ID"; + final String targetNameHeader = "TARGET_NAME"; + final String outcomeNameHeader = "OUTCOME_NAME"; final String personCountHeader = "PERSON_COUNT"; final String timeAtRiskHeader = "TIME_AT_RISK"; final String casesHeader = "CASES"; Map values = - Arrays.asList(personCountHeader, timeAtRiskHeader, casesHeader) - .stream() + Stream.of(targetIdHeader, outcomeIdHeader, personCountHeader, timeAtRiskHeader, casesHeader) .collect(Collectors.toMap(header -> header, headers::get)); final List records = parser.getRecords(); if (!CollectionUtils.isEmpty(records)) { - final CSVRecord firstRecord = records.get(0); - final String personCount = firstRecord.get(values.get(personCountHeader)); - final String timeAtRisk = firstRecord.get(values.get(timeAtRiskHeader)); - final String cases = firstRecord.get(values.get(casesHeader)); - - resultInfo.add(personCountHeader, getJsonPrimitive(personCount)); - resultInfo.add(timeAtRiskHeader, getJsonPrimitive(timeAtRisk)); - resultInfo.add(casesHeader, getJsonPrimitive(cases)); - try { - final float casesFloat = cast(cases).floatValue(); + records.forEach(record -> { + + final String targetId = record.get(values.get(targetIdHeader)); + final String targetName = cohortNames.getOrDefault(targetId, ""); + final String outcomeId = record.get(values.get(outcomeIdHeader)); + final String outcomeName = cohortNames.getOrDefault(outcomeId, ""); + final String personCount = record.get(values.get(personCountHeader)); + final String timeAtRisk = record.get(values.get(timeAtRiskHeader)); + final String cases = record.get(values.get(casesHeader)); + + final JsonObject resultInfo = new JsonObject(); + resultInfo.add(targetIdHeader, getJsonPrimitive(targetId)); + resultInfo.add(targetNameHeader, getJsonPrimitive(targetName)); + resultInfo.add(outcomeIdHeader, getJsonPrimitive(outcomeId)); + resultInfo.add(outcomeNameHeader, getJsonPrimitive(outcomeName)); + resultInfo.add(personCountHeader, getJsonPrimitive(personCount)); + resultInfo.add(timeAtRiskHeader, getJsonPrimitive(timeAtRisk)); + resultInfo.add(casesHeader, getJsonPrimitive(cases)); try { - final float timeAtRiskFloat = cast(timeAtRisk).floatValue(); - final float rate = timeAtRiskFloat > 0 ? casesFloat / timeAtRiskFloat * 1000 : 0F; - resultInfo.add("RATE", new JsonPrimitive(rate)); + final float casesFloat = cast(cases).floatValue(); + try { + final float timeAtRiskFloat = cast(timeAtRisk).floatValue(); + final float rate = timeAtRiskFloat > 0 ? casesFloat / timeAtRiskFloat * 1000 : 0F; + resultInfo.add("RATE", new JsonPrimitive(rate)); + } catch (IllegalArgumentException e) { + LOGGER.debug("'TIME_AT_RISK' is not correct value, skipping calculate 'RATE' value"); + } + try { + final float personsFloat = cast(personCount).floatValue(); + final float proportion = personsFloat > 0 ? casesFloat / personsFloat * 1000 : 0F; + resultInfo.add("PROPORTION", new JsonPrimitive(proportion)); + } catch (IllegalArgumentException e) { + LOGGER.debug("'TIME_AT_RISK' is not correct value, skipping calculate 'PROPORTION' value"); + } + result.add(resultInfo); } catch (IllegalArgumentException e) { - LOGGER.debug("'TIME_AT_RISK' is not correct value, skipping calculate 'RATE' value"); + LOGGER.debug("'PERSON_COUNT' is not correct value, skipping calculate 'RATE' & 'PROPORTION' values"); } - try { - final float personsFloat = cast(personCount).floatValue(); - final float proportion = personsFloat > 0 ? casesFloat / personsFloat * 1000 : 0F; - resultInfo.add("PROPORTION", new JsonPrimitive(proportion)); - } catch (IllegalArgumentException e) { - LOGGER.debug("'TIME_AT_RISK' is not correct value, skipping calculate 'PROPORTION' value"); - } - } catch (IllegalArgumentException e) { - LOGGER.debug("'PERSON_COUNT' is not correct value, skipping calculate 'RATE' & 'PROPORTION' values"); - } + }); } } catch (IOException e) { @@ -262,7 +341,18 @@ public void updateExtendInfo(Submission submission) { LOGGER.warn(CAN_NOT_BUILD_EXTEND_INFO_LOG, submission.getId()); LOGGER.warn("Error: ", e); } - submission.setResultInfo(resultInfo); + submission.setResultInfo(result); + } + + private Map getCohortNames(JsonArray cohorts) { + + Map result = new HashMap<>(); + cohorts.forEach(c -> { + String id = c.getAsJsonObject().get("id").getAsString(); + String name = c.getAsJsonObject().get("name").getAsString(); + result.put(id, name); + }); + return result; } } diff --git a/src/main/java/com/odysseusinc/arachne/portal/util/UUIDGenerator.java b/src/main/java/com/odysseusinc/arachne/portal/util/UUIDGenerator.java deleted file mode 100644 index 3c8f29049..000000000 --- a/src/main/java/com/odysseusinc/arachne/portal/util/UUIDGenerator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * - * Copyright 2018 Odysseus Data Services, inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Company: Odysseus Data Services, Inc. - * Product Owner/Architecture: Gregory Klebanov - * Authors: Pavel Grafkin, Alexandr Ryabokon, Vitaly Koulakov, Anton Gackovka, Maria Pozhidaeva, Mikhail Mironov - * Created: September 14, 2017 - * - */ - -package com.odysseusinc.arachne.portal.util; - -import java.util.UUID; - -public class UUIDGenerator { - - private UUIDGenerator() { - - } - - public static String generateUUID() { - - return UUID.randomUUID().toString().replace("-", ""); - } -} diff --git a/src/main/resources/application-base.yml b/src/main/resources/application-base.yml index abdbe8229..c79e88634 100644 --- a/src/main/resources/application-base.yml +++ b/src/main/resources/application-base.yml @@ -83,11 +83,13 @@ flyway: #locations: db/migration,classpath:com.odysseusinc.arachne.portal.db.migration arachne: token: - secret: sssshhhh! + secret: 129DF19C8A91AFD8375A2826A33539K01ACQ778QOJFAA9MGWLWH73PLXVFVHBR7860MTIE2O8EEVF9KCO77P6A7NUNX4XHAGCRFSBWG879XPDOIN6C2LFCKJI002OIABS4D6Q9VMJJIX8UCE48EF header: Arachne-Auth-Token expiration: 900 systemToken: header: Arachne-System-Token + impersonate: + header: Arachne-Auth-Impersonate resetPasswordToken: expiresMinutes: 60 loginAttempts: @@ -146,10 +148,32 @@ jodconverter: antivirus: host: localhost port: 3310 + executor: + queueCapacity: 50 retry: max-attempts: 10 backoff: max-interval: 50000 tmp: holder: - cron: 0 0 6 * * ? \ No newline at end of file + cron: 0 0 6 * * ? +authenticator: + methods: + db: + service: org.ohdsi.authenticator.service.jdbc.JdbcAuthService + config: + jdbcUrl: ${spring.datasource.url} + username: ${spring.datasource.username} + password: ${spring.datasource.password} + query: SELECT password, firstname, middlename, lastname FROM users_data WHERE email = :username + passwordEncoder: org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder + fieldsToExtract: + firstName: firstname + middleName: middlename + lastName: lastname +security: + method: db + jwt: + token: + secretKey: ${arachne.token.secret} + validityInSeconds: ${arachne.token.expiration} \ No newline at end of file diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 9ab684755..e1824d83d 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -6,7 +6,9 @@ spring: username: ohdsi password: ENC(0Lpfvg9UPAyaaZpSIqwaDg==) logging: - level: debug + level: + root: INFO + com.odysseusinc: DEBUG portal: urlWhiteList: https://localhost:${server.port} build: diff --git a/src/main/resources/r/ir/additionalCriteria.sql b/src/main/resources/r/ir/additionalCriteria.sql deleted file mode 100644 index 981e86143..000000000 --- a/src/main/resources/r/ir/additionalCriteria.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Begin Correlated Criteria -SELECT @indexId as index_id, p.person_id, p.event_id -FROM @eventTable P -LEFT JOIN -( - @criteriaQuery -) A on A.person_id = P.person_id and @windowCriteria -GROUP BY p.person_id, p.event_id -@occurrenceCriteria --- End Correlated Criteria \ No newline at end of file diff --git a/src/main/resources/r/ir/analysis_summary.sql b/src/main/resources/r/ir/analysis_summary.sql deleted file mode 100644 index 4e09c708d..000000000 --- a/src/main/resources/r/ir/analysis_summary.sql +++ /dev/null @@ -1,10 +0,0 @@ -select - target_id, outcome_id, sum(person_count) as person_count, - sum(time_at_risk) as time_at_risk, - sum(cases) as cases -from - @resultsSchema.ir_analysis_result -where - analysis_id = @id -GROUP BY - target_id, outcome_id \ No newline at end of file diff --git a/src/main/resources/r/ir/delete_strata.sql b/src/main/resources/r/ir/delete_strata.sql deleted file mode 100644 index 9b9216e3b..000000000 --- a/src/main/resources/r/ir/delete_strata.sql +++ /dev/null @@ -1 +0,0 @@ -DELETE FROM @tableQualifier.ir_strata WHERE analysis_id = @analysis_id \ No newline at end of file diff --git a/src/main/resources/r/ir/groupQuery.sql b/src/main/resources/r/ir/groupQuery.sql deleted file mode 100644 index 92cb8a4f6..000000000 --- a/src/main/resources/r/ir/groupQuery.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Begin Criteria Group -select @indexId as index_id, person_id, event_id -FROM -( - select E.person_id, E.event_id - FROM @eventTable E - LEFT JOIN - ( - @criteriaQueries - ) CQ on E.person_id = CQ.person_id and E.event_id = CQ.event_id - GROUP BY E.person_id, E.event_id - @occurrenceCountClause -) G --- End Criteria Group \ No newline at end of file diff --git a/src/main/resources/r/ir/ir_analysis_query_builder.r b/src/main/resources/r/ir/ir_analysis_query_builder.r deleted file mode 100644 index 4dae6aea4..000000000 --- a/src/main/resources/r/ir/ir_analysis_query_builder.r +++ /dev/null @@ -1,953 +0,0 @@ -library(rJava) - -getCorelatedCriteriaQuery <- function(corelatedCriteria, eventTable, dbms){ - sql <- SqlRender::readSql("additionalQuery.sql") - sql <- SqlRender::renderSql(sql)$sql - sql <- SqlRender::translateSql(sql, targetDialect = dbms)$sql - return(sql) -} - -getCriteriaGroupQuery <- function(group, eventTable, dbms){ - sql <- SqlRender::readSql("groupQuery.sql") - - additionalCriteriaQueries <- c() - for(i in seq_along(group$CriteriaList)){ - cc <- group$CriteriaList[i] - sql <- getCorelatedCriteriaQuery(cc, eventTable, dbms) - sql <- gsub("@indexId", i, sql) - additionalCriteriaQueries[[i]] <- sql - } - n <- length(additionalCriteriaQueries) - for(i in seq_along(group$DemographicCriteriaList)){ - dc <- group$DemographicCriteriaList[[i]] - sql <- getDemographicCriteriaQuery(dc, eventTable, dbms) - sql <- gsub("@indexId", i + n, sql) - additionalCriteriaQueries[[i + n]] <- sql - } - n <- length(additionalCriteriaQueries) - for(i in seq_along(group$Groups)){ - g <- group$Groups[[i]] - sql <- getCriteriaGroupQuery(g, eventTable, dbms) - sql <- gsub("@indexId", i + n, sql) - additionalCriteriaQueries[[i + n]] <- sql - } - - if (length(additionalCriteriaQueries) > 0){ - sql <- gsub("@criteriaQueries", paste(additionalCriteriaQueries, collapse = "\nUNION ALL\n")) - } - - sql <- SqlRender::renderSql(sql)$sql - sql <- SqlRender::translateSql(sql, targetDialect = dbms)$sql - return(sql) -} - -convertWindowEndpoint <- function(endpoint){ - w <- .jnew("org/ohdsi/circe/cohortdefinition/Endpoint") - if (!is.null(endpoint$Days)){ - days <- .jnew("java/lang/Integer", toString(endpoint$Days)) - `.jfield<-`(w, 'days', days) - } - if (!is.null(endpoint$Coeff)){ - `.jfield<-`(w, 'coeff', as.integer(endpoint$Coeff)) - } - return(w) -} - -convertWindow <- function(window){ - w <- .jnew("org/ohdsi/circe/cohortdefinition/Window") - start <- convertWindowEndpoint(window$Start) - `.jfield<-`(w, 'start', start) - end <- convertWindowEndpoint(window$End) - `.jfield<-`(w, 'end', end) - return(w) -} - -convertConcept <- function(concept){ - c <- .jnew("org/ohdsi/circe/vocabulary/Concept") - conceptId <- .jnew("java/lang/Long", toString(concept$CONCEPT_ID)) - `.jfield<-`(c, 'conceptId', conceptId) - `.jfield<-`(c, 'conceptName', concept$CONCEPT_NAME) - `.jfield<-`(c, 'standardConcept', concept$STANDARD_CONCEPT) - `.jfield<-`(c, 'invalidReason', concept$INVALID_REASON) - `.jfield<-`(c, 'conceptCode', concept$CONCEPT_CODE) - `.jfield<-`(c, 'domainId', toString(concept$DOMAIN_ID)) - `.jfield<-`(c, 'vocabularyId', toString(concept$VOCABULARY_ID)) - `.jfield<-`(c, 'conceptClassId', toString(concept$CONCEPT_CLASS_ID)) - return(c) -} - -convertConceptArray <- function(concepts){ - cc <- c() - for(i in seq_along(concepts)){ - concept <- concepts[[i]] - cc[[i]] <- convertConcept(concept) - } - return(.jarray(cc, contents.class = 'org/ohdsi/circe/vocabulary/Concept')) -} - -convertDateRange <- function(dateRange){ - dr <- .jnew("org/ohdsi/circe/cohortdefinition/DateRange") - `.jfield<-`(dr, 'value', dateRange$Value) - `.jfield<-`(dr, 'op', dateRange$Op) - `.jfield<-`(dr, 'extent', dateRange$Extent) - return(dr) -} - -convertPeriod <- function(period){ - p <- .jnew("org/ohdsi/circe/cohortdefinition/Period") - if (!is.null(period$StartDate)){ - `.jfield<-`(p, 'startDate', period$StartDate) - } - if (!is.null(period$EndDate)){ - `.jfield<-`(p, 'periodEndDate', period$EndDate) - } -} - -convertNumericRange <- function(range){ - r <- .jnew("org/ohdsi/circe/cohortdefinition/NumericRange") - if (!is.null(range$Value)){ - value <- .jnew("java/lang/Integer", toString(range$Value)) - `.jfield<-`(r, 'value', value) - } - if (!is.null(range$Op)){ - `.jfield<-`(r, 'op', range$Op) - } - if (!is.null(range$Extent)){ - extent <- .jnew("java/lang/Integer") - `.jfield<-`(r, 'extent', extent) - } - return(r) -} - -convertTextFilter <- function(filter){ - tf <- .jnew("org/ohdsi/circe/cohortdefinition/TextFilter") - if (!is.null(filter$Text)){ - `.jfield<-`(tf, 'text', filter$Text) - } - if (!is.null(filter$Op)){ - `.jfield<-`(tf, 'op', filter$Op) - } - return(tf) -} - -convertCriteria <- function(criteria){ - c <- NULL - if (!is.null(criteria$ConditionEra)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/ConditionEra") - if (!is.null(criteria$ConditionEra$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(criteria$ConditionEra$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(criteria$ConditionEra$First))) - `.jfield<-`(c, 'first', first) - if (!is.null(criteria$ConditionEra$EraStartDate)){ - eraStartDate <- convertDateRange(criteria$ConditionEra$EraStartDate) - `.jfield<-`(c, 'eraStartDate', eraStartDate) - } - if (!is.null(criteria$ConditionEra$EraEndDate)){ - eraEndDate <- convertDateRange(criteria$ConditionEra$EraEndDate) - `.jfield<-`(c, 'eraEndDate', eraEndDate) - } - if (!is.null(criteria$ConditionEra$OccurrenceCount)){ - occurrenceCount <- convertNumericRange(criteria$ConditionEra$OccurrenceCount) - `.jfield<-`(c, 'occurrenceCount', occurrenceCount) - } - if (!is.null(criteria$ConditionEra$EraLength)){ - eraLength <- convertNumericRange(criteria$ConditionEra$EraLength) - `.jfield<-`(c, 'eraLength', eraLength) - } - if (!is.null(criteria$ConditionEra$AgeAtStart)){ - ageAtStart <- convertNumericRange(criteria$ConditionEra$AgeAtStart) - `.jfield<-`(c, 'ageAtStart', ageAtStart) - } - if (!is.null(criteria$ConditionEra$AgeAtEnd)){ - ageAtEnd <- convertNumericRange(criteria$ConditionEra$AgeAtEnd) - `.jfield<-`(c, 'ageAtEnd', ageAtEnd) - } - if (!is.null(critera$ConditionEra$Gender)){ - jgArray = convertConceptArray(criteria$ConditionEra$Gender) - `.jfield<-`(c, 'gender', jgArray) - } - } else if (!is.null(criteria$ConditionOccurrence)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/ConditionOccurrence") - conditionOccurrence <- criteria$ConditionOccurrence - if (!is.null(conditionOccurrence$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(conditionOccurrence$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(conditionOccurrence$First))) - `.jfield<-`(c, 'first', first) - if (!is.null(conditionOccurrence$OccurrenceStartDate)){ - occurrenceStartDate <- convertDateRange(conditionOccurrence$OccurrenceStartDate) - `.jfield<-`(c, 'occurrenceStartDate', occurrenceStartDate) - } - if (!is.null(conditionOccurrence$OccurrenceEndDate)){ - occurrenceEndDate <- convertDateRange(conditionOccurrence$OccurrenceEndDate) - `.jfield<-`(c, 'occurrenceEndDate', occurrenceEndDate) - } - if (!is.null(conditionOccurrence$ConditionType)){ - conditionTypes <- list() - for(i in seq_along(conditionOccurrence$ConditionType)){ - type <- conditionOccurrence$ConditionType[[i]] - jtype <- convertConcept(type) - conditionTypes[[i]] <- jtype - } - jArray <- .jarray(conditionTypes, contents.class = "org/ohdsi/circe/vocabulary/Concept") - `.jfield<-`(c, 'conditionType', jArray) - } - if (!is.null(conditionOccurrence$StopReason)){ - stopReason <- convertTextFilter(conditionOccurrence$StopReason) - `.jfield<-`(c, 'stopReason', stopReason) - } - if (!is.null(conditionOccurrence$ConditionSourceConcept)){ - conditionSourceConcept <- .jnew("java/lang/Integer", toString(conditionOccurrence$ConditionSourceConcept)) - `.jfield<-`(c, 'conditionSourceConcept', conditionSourceConcept) - } - if (!is.null(conditionOccurrence$Age)){ - age <- convertNumericRange(conditionOccurrence$Age) - `.jfield<-`(c, 'age', age) - } - if (!is.null(conditionOccurrence$Gender)){ - genders <- list() - for(i in seq_along(conditionOccurrence$Gender)){ - g <- conditionOccurrence$Gender[[i]] - jg <- convertConcept(g) - genders[[i]] <- g - } - jgArray = .jarray(genders, contents.class = "org/ohdsi/circe/vocabulary/Concept") - `.jfield<-`(c, 'gender', jgArray) - } - if (!is.null(conditionOccurrence$ProviderSpecialty)){ - jArray <- convertConceptArray(conditionOccurrence$ProviderSpeciality) - `.jfield<-`(c, 'providerSpecialty', jArray) - } - if (!is.null(conditionOccurrence$VisitType)){ - jArray <- convertConceptArray(conditionOccurrence$VisitType) - `.jfield<-`(c, 'visitType', jArray) - } - } else if (!is.null(criteria$Death)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/Death") - death <- criteria$Death - if (!is.null(death$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(death$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - if (!is.null(death$OccurrenceStartDate)){ - occurrenceStartDate <- convertDateRange(death$OccurrenceStartDate) - `.jfield<-`(c, "occurrenceStartDate", occurrenceStartDate) - } - if (!is.null(death$DeathType)){ - jArray <- convertConceptArray(death$DeathType) - `.jfield<-`(c, 'deathType', jArray) - } - if (!is.null(death$DeathSourceConcept)){ - sourceConcept <- .jnew("java/lang/Integer", toString(death$DeathSourceConcept)) - `.jfield<-`(c, 'deathSourceConcept', sourceConcept) - } - if (!is.null(death$Age)){ - age <- convertNumericRange(death$Age) - `.jfield<-`(c, 'age', age) - } - if (!is.null(death$Gender)){ - jArray <- convertConceptArray(death$Gender) - `.jfield<-`(c, 'gender', jArray) - } - } else if (!is.null(criteria$DeviceExposure)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/DeviceExposure") - deviceExposure <- criteria$DeviceExposure - if (!is.null(deviceExposure$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(deviceExposure$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(deviceExposure$First))) - `.jfield<-`(c, 'first', first) - if (!is.null(deviceExposure$OccurrenceStartDate)){ - occurrenceStartDate <- convertDateRange(deviceExposure$OccurrenceStartDate) - `.jfield<-`(c, 'occurrenceStartDate', occurrenceStartDate) - } - if (!is.null(deviceExposure$OccurrenceEndDate)){ - occurrenceEndDate <- convertDateRange(deviceExposure$OccurrenceEndDate) - `.jfield<-`(c, 'occurrenceEndDate', occurrenceEndDate) - } - if (!is.null(deviceExposure$DeviceType)){ - jArray <- convertConceptArray(deviceExposure$DeviceType) - `.jfield<-`(c, 'deviceType', jArray) - } - if (!is.null(deviceExposure$UniqueDeviceId)){ - uniqueDeviceId <- convertTextFilter(deviceExposure$UniqueDeviceId) - `.jfield<-`(c, 'uniqueDeviceId', uniqueDeviceId) - } - if (!is.null(deviceExposure$Quantity)){ - quantity <- convertNumericRange(deviceExposure$Quantity) - `.jfield<-`(c, 'quantity', quantity) - } - if (!is.null(deviceExposure$DeviceSourceConcept)){ - deviceSourceConcept <- .jnew("java/lang/Integer", toString(deviceExposure$DeviceSourceConcept)) - `.jfield<-`(c, 'deviceSourceConcept', deviceSourceConcept) - } - if (!is.null(deviceExposure$Age)){ - age <- convertNumericRange(deviceExposure$Age) - `.jfield<-`(c, 'age', age) - } - if (!is.null(deviceExposure$Gender)){ - jArray <- convertConceptArray(deviceExposure$Gender) - `.jfield<-`(c, 'gender', jArray) - } - if (!is.null(deviceExposure$ProviderSpecialty)){ - jArray <- convertConceptArray(deviceExposure$ProviderSpecialty) - `.jfield<-`(c, 'providerSpeciality', jArray) - } - if (!is.null(deviceExposure$VisitType)){ - jArray <- convertConceptArray(deviceExposure$VisitType) - `.jfield<-`(c, 'visitType', jArray) - } - } else if (!is.null(criteria$DoseEra)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/DoseEra") - doseEra <- criteria$DoseEra - if (!is.null(doseEra$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(doseEra$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(doseEra$First))) - `.jfield<-`(c, 'first', first) - if (!is.null(doseEra$EraStartDate)){ - eraStartDate <- convertDateRange(doseEra$EraStartDate) - `.jfield<-`(c, 'eraStartDate', eraStartDate) - } - if (!is.null(doseEra$EraEndDate)){ - eraEndDate <- convertDateRange(doseEra$EraStartDate) - `.jfield<-`(c, 'eraEndDate', eraEndDate) - } - if (!is.null(doseEra$Unit)){ - jArray <- convertConceptArray(doseEra$Unit) - `.jfield<-`(c, 'unit', jArray) - } - if (!is.null(doseEra$DoseValue)){ - doseValue <- convertNumericRange(doseEra$DoseValue) - `.jfield<-`(c, 'doseValue', doseValue) - } - if (!is.null(doseEra$EraLength)){ - eraLength <- convertNumericRange(doseEra$EraLength) - `.jfield<-`(c, 'eraLength', eraLength) - } - if (!is.null(doseEra$AgeAtStart)){ - ageAtStart <- convertNumericRange(doseEra$AgeAtStart) - `.jfield<-`(c, 'ageAtStart', ageAtStart) - } - if (!is.null(doseEra$AgeAtEnd)){ - ageAtEnd <- convertNumericRange(doseEra$AgeAtEnd) - `.jfield<-`(c, 'ageAtEnd', ageAtEnd) - } - if (!is.null(doseEra$Gender)){ - jArray <- convertConceptArray(doseEra$Gender) - `.jfield<-`(c, 'gender', jArray) - } - } else if (!is.null(criteria$DrugEra)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/DrugEra") - drugEra <- criteria$DrugEra - if (!is.null(doseEra$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(drugEra$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(drugEra$First))) - `.jfield<-`(c, 'first', first) - if (!is.null(drugEra$EraStartDate)){ - eraStartDate <- convertDateRange(drugEra$EraStartDate) - `.jfield<-`(c, 'eraStartDate', eraStartDate) - } - if (!is.null(drugEra$EraEndDate)){ - eraEndDate <- convertDateRange(drugEra$EraEndDate) - `.jfield<-`(c, "eraEndDate", eraEndDate) - } - if (!is.null(drugEra$OccurrenceCount)){ - occurrenceCount <- convertNumericRange(drugEra$OccurrenceCount) - `.jfield<-`(c, 'occurrenceCount', occurrenceCount) - } - if (!is.null(drugEra$GapDays)){ - gapDays <- convertNumericRange(drugEra$GapDays) - `.jfield<-`(c, 'gapDays', gapDays) - } - if (!is.null(drugEra$EraLength)){ - eraLength <- convertNumericRange(drugEra$EraLength) - `.jfield<-`(c, 'eraLength', eraLength) - } - if (!is.null(drugEra$AgeAtStart)){ - ageAtStart <- convertNumericRange(drugEra$AgeAtStart) - `.jfield<-`(c, 'ageAtStart', ageAtStart) - } - if (!is.null(drugEra$AgeAtEnd)){ - ageAtEnd <- convertNumericRange(drugEra$AgeAtEnd) - `.jfield<-`(c, 'ageAtEnd', ageAtEnd) - } - if (!is.null(drugEra$Gender)){ - jArray <- convertConceptArray(drugEra$Gender) - `.jfield<-`(c, 'gender', jArray) - } - } else if (!is.null(criteria$DrugExposure)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/DrugExposure") - drugExposure <- criteria$DrugExposure - if (!is.null(drugExposure$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(drugExposure$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(drugExposure$First))) - `.jfield<-`(c, 'first', first) - if (!is.null(drugExposure$OccurrenceStartDate)){ - occurrenceStartDate <- convertDateRange(drugExposure$OccurrenceStartDate) - `.jfield<-`(c, 'occurrenceStartDate', occurrenceStartDate) - } - if (!is.null(drugExposure$OccurrenceEndDate)){ - occurrenceEndDate <- convertDateRange(drugExposure$OccurrenceEndDate) - `.jfield<-`(c, 'occurrenceEndDate', occurrenceEndDate) - } - if (!is.null(drugExposure$StopReason)){ - stopReason <- convertTextFilter(drugExposure$StopReason) - `.jfield<-`(c, 'stopReason', stopReason) - } - if (!is.null(drugExposure$Refills)){ - refills <- convertNumericRange(drugExposure$Refills) - `.jfield<-`(c, 'refills', refills) - } - if (!is.null(drugExposure$Quantity)){ - quantity <- convertNumericRange(drugExposure$Quantity) - `.jfield<-`(c, 'quantity', quantity) - } - if (!is.null(drugExposure$DaysSupply)){ - daysSupply <- convertNumericRange(drugExposure$daysSupply) - `.jfield<-`(c, 'daysSupply', daysSupply) - } - if (!is.null(drugExposure$RouteConcept)){ - jArray <- convertConceptArray(drugExposure$RouteConcept) - `.jfield<-`(c, 'routeConcept', jArray) - } - if (!is.null(drugExposure$EffectiveDrugDose)){ - effectiveDrugDose <- convertNumericRange(drugExposure$EffectiveDrugDose) - `.jfield<-`(c, 'effectiveDrugDose', effectiveDrugDose) - } - if (!is.null(drugExposure$DoseUnit)){ - jArray <- convertConceptArray(drugExposure$DoseUnit) - `.jfield<-`(c, 'doseUnit', doseUnit) - } - if (!is.null(drugExposure$LotNumber)){ - lotNumber <- convertTextFilter(drugExposure$LotNumber) - `.jfield<-`(c, 'lotNumber', lotNumber) - } - if (!is.null(drugExposure$DrugSourceConcept)){ - drugSourceConcept <- .jnew("java/lang/Integer", toString(drugExposure$DrugSourceConcept)) - `.jfield<-`(c, 'drugSourceConcept', drugSourceConcept) - } - if (!is.null(drugExposure$Age)){ - age <- convertNumericRange(drugExposure$Age) - `.jfield<-`(c, 'age', age) - } - if (!is.null(drugExposure$Gender)){ - jArray <- convertConceptArray(drugExposure$Gender) - `.jfield<-`(c, 'gender', gender) - } - if (!is.null(drugExposure$ProviderSpecialty)){ - jArray <- convertConceptArray(drugExposure$ProviderSpecialty) - `.jfield<-`(c, 'providerSpecialty', drugExposure$ProviderSpecialty) - } - if (!is.null(drugExposure$VistType)){ - jArray <- convertConceptArray(drugExposure$VisitType) - `.jfield<-`(c, 'visitType', jArray) - } - } else if (!is.null(criteria$Measurement)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/Measurement") - measurement <- criteria$Measurement - if (!is.null(measurement$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(measurement$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(measurement$First))) - `.jfield<-`(c, 'first', first) - if (!is.null(measurement$OccurrenceStartDate)){ - occurrenceStartDate <- convertDateRange(measurement$OccurrenceStartDate) - `.jfield<-`(c, 'occurrenceStartDate', occurrenceStartDate) - } - if (!is.null(measurement$MeasurementType)){ - jArray <- convertConceptArray(measurement$MeasurementType) - `.jfield<-`(c, 'measurementType', jArray) - } - if (!is.null(measurement$Operator)){ - jArray <- convertConceptArray(measurement$Operator) - `.jfield<-`(c, 'operator', jArray) - } - if (!is.null(measurement$ValueAsNumber)){ - valueAsNumber <- convertNumericRange(measurement$ValueAsNumber) - `.jfield<-`(c, 'valueAsNumber', valueAsNumber) - } - if (!is.null(measurement$ValueAsConcept)){ - jArray <- convertConceptArray(measurement$ValueAsConcept) - `.jfield<-`(c, 'valueAsConcept', jArray) - } - if (!is.null(measurement$Unit)){ - jArray <- convertConceptArray(measurement$Unit) - `.jfield<-`(c, 'unit', jArray) - } - if (!is.null(measurement$RangeLow)){ - rangeLow <- convertNumericRange(measurement$RangeLow) - `.jfield<-`(c, 'rangeLow', rangeLow) - } - if (!is.null(measurement$RangeHigh)){ - rangeHigh <- convertNumericRange(measurement$RangeHigh) - `.jfield<-`(c, 'rangeHigh', rangeHigh) - } - if (!is.null(measurement$RangeLowRatio)){ - rangeLowRatio <- convertNumericRange(measurement$RangeLowRatio) - `.jfield<-`(c, 'rangeLowRatio', rangeLowRatio) - } - if (!is.null(measurement$RangeHighRatio)){ - rangeHighRatio <- convertNumericRange(measurement$RangeHighRatio) - `.jfield<-`(c, 'rangeHighRatio', rangeHighRatio) - } - abnormal <- .jnew("java/lang/Boolean", toString(isTRUE(measurement$Abnormal))) - .jfield(c, 'abnormal', abnormal) - if (!is.null(measurement$MeasurementSourceConcept)){ - measurementSourceConcept <- .jnew("java/lang/Integer", toString(measurement$MeasurementSourceConcept)) - `.jfield<-`(c, 'measurementSourceConcept', measurementSourceConcept) - } - if (!is.null(measurement$Age)){ - age <- convertNumericRange(measurement$Age) - `.jfield<-`(c, 'age', age) - } - if (!is.null(measurement$Gender)){ - jArray <- convertConceptArray(measurement$Gender) - `.jfield<-`(c, 'gender', jArray) - } - if (!is.null(measurement$ProviderSpecialty)){ - jArray <- convertConceptArray(measurement$ProviderSpecialty) - `.jfield<-`(c, 'providerSpecialty', jArray) - } - if (!is.null(measurement$VisitType)){ - jArray <- convertConceptArray(measurement$VisitType) - `.jfield<-`(c, 'visitType', jArray) - } - } else if (!is.null(criteria$Observation)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/Observation") - observation <- criteria$Observation - if (!is.null(observation$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(observation$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(observation$First))) - if (!is.null(observation$OccurrenceStartDate)){ - occurrenceStartDate <- observation$OccurrenceStartDate - `.jfield<-`(c, 'occurrenceStartDate', occurrenceStartDate) - } - if (!is.null(observation$ObservationType)){ - jArray <- convertConceptArray(observation$ObservationType) - `.jfield<-`(c, 'observationType', jArray) - } - if (!is.null(observation$ValueAsNumber)){ - valueAsNumber <- convertNumericRange(observation$ValueAsNumber) - `.jfield<-`(c, 'valueAsNumber', valueAsNumber) - } - if (!is.null(observation$ValueAsString)){ - valueAsString <- convertTextFilter(observation$ValueAsString) - `.jfield<-`(c, 'valueAsString', valueAsString) - } - if (!is.null(observation$ValueAsConcept)){ - jArray <- convertConceptArray(observation$ValueAsConcept) - `.jfield<-`(c, 'valueAsConcept', jArray) - } - if (!is.null(observation$Qualifier)){ - qualifier <- convertConceptArray(observation$qualifier) - `.jfield<-`(c, 'qualifier', qualifier) - } - if (!is.null(observation$Unit)){ - jArray <- convertConceptArray(observation$Unit) - `.jfield<-`(c, 'unit', jArray) - } - if (!is.null(observation$ObservationSourceConcept)){ - conceptId <- .jnew("java/lang/Integer", toString(observation$ObservationSourceConcept)) - `.jfield<-`(c, 'observationSourceConcept', conceptId) - } - if (!is.null(observation$Age)){ - age <- convertNumericRange(observation$Age) - `.jfield<-`(c, 'age', age) - } - if (!is.null(observation$Gender)){ - jArray <- convertConceptArray(observation$Gender) - `.jfield<-`(c, 'gender', jArray) - } - if (!is.null(measurement$ProviderSpecialty)){ - jArray <- convertConceptArray(measurement$ProviderSpecialty) - `.jfield<-`(c, 'providerSpecialty', jArray) - } - if (!is.null(measurement$VisitType)){ - jArray <- convertConceptArray(measurement$VisitType) - `.jfield<-`(c, 'visitType', jArray) - } - } else if (!is.null(criteria$ObservationPeriod)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/ObservationPeriod") - observationPeriod <- criteria$ObservationPeriod - first <- .jnew("java/lang/Boolean", toString(isTRUE(observationPeriod$First))) - if (!is.null(observationPeriod$PeriodStartDate)){ - periodStartDate <- convertDateRange(observationPeriod$PeriodStartDate) - `.jfield<-`(c, 'periodStartDate', periodStartDate) - } - if (!is.null(observationPeriod$PeriodEndDate)){ - periodEndDate <- convertDateRange(observationPeriod$PeriodEndDate) - `.jfield<-`(c, 'periodEndDate', periodEndDate) - } - if (!is.null(observationPeriod$UserDefinedPeriod)){ - udp <- convertPeriod(observationPeriod$UserDefinedPeriod) - `.jfield<-`(c, 'userDefinedPeriod', udp) - } - if (!is.null(observationPeriod$PeriodType)){ - jArray <- convertConceptArray(observationPeriod$PeriodType) - `.jfield<-`(c, 'periodType', jArray) - } - if (!is.null(observationPeriod$PeriodLength)){ - periodLength <- convertNumericRange(observationPeriod$PeriodLength) - `.jfield<-`(c, 'periodLength', periodLength) - } - if (!is.null(observationPeriod$AgeAtStart)){ - ageAtStart <- convertNumericRange(observationPeriod$AgeAtStart) - `.jfield<-`(c, 'ageAtStart', ageAtStart) - } - if (!is.null(observationPeriod$AgeAtEnd)){ - ageAtEnd <- convertNumericRange(observationPeriod$AgeAtEnd) - `.jfield<-`(c, 'ageAtEnd', ageAtEnd) - } - } else if (!is.null(criteria$ProcedureOccurrence)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/ProcedureOccurrence") - procedureOccurrence <- criteria$ProcedureOccurrence - first <- .jnew("java/lang/Boolean", toString(isTRUE(procedureOccurrence$First))) - if (!is.null(procedureOccurrence$PeriodStartDate)){ - periodStartDate <- convertDateRange(procedureOccurrence$PeriodStartDate) - `.jfield<-`(c, 'periodStartDate', periodStartDate) - } - if (!is.null(procedureOccurrence$PeriodEndDate)){ - periodEndDate <- convertDateRange(procedureOccurrence$PeriodEndDate) - `.jfield<-`(c, 'periodEndDate', periodEndDate) - } - if (!is.null(procedureOccurrence$PeriodType)){ - jArray <- convertConceptArray(procedureOccurrence$PeriodType) - `.jfield<-`(c, 'periodType', jArray) - } - if (!is.null(procedureOccurrence$PeriodLength)){ - periodLength <- convertNumericRange(procedureOccurrence$PeriodLength) - `.jfield<-`(c, 'periodLength', periodLength) - } - if (!is.null(procedureOccurrence$AgeAtStart)){ - ageAtStart <- convertNumericRange(procedureOccurrence$AgeAtStart) - `.jfield<-`(c, 'ageAtStart', ageAtStart) - } - if (!is.null(procedureOccurrence$AgeAtEnd)){ - ageAtEnd <- convertNumericRange(procedureOccurrence$AgeAtEnd) - `.jfield<-`(c, 'ageAtEnd', ageAtEnd) - } - } else if (!is.null(criteria$Specimen)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/Specimen") - specimen <- criteria$Specimen - if (!is.null(specimen$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(specimen$CodesetId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", toString(isTRUE(specimen$First))) - if (!is.null(specimen$OccurrenceStartDate)){ - occurrenceStartDate <- convertDateRange(specimen$OccurrenceStartDate) - `.jfield<-`(c, 'occurrenceStartDate', occurrenceStartDate) - } - if (!is.null(specimen$SpecimenType)){ - jArray <- convertConceptArray(specimen$SpecimenType) - `.jfield<-`(c, 'specimenType', jArray) - } - if (!is.null(specimen$Quantity)){ - quantity <- convertNumericRange(specimen$Quantity) - `.jfield<-`(c, 'quantity', quantity) - } - if (!is.null(specimen$Unit)){ - jArray <- convertConceptArray(specimen$Unit) - `.jfield<-`(c, 'unit', jArray) - } - if (!is.null(specimen$AnatomicSite)){ - jArray <- convertConceptArray(specimen$AnatomicSite) - `.jfield<-`(c, 'anatomicSite', specimen$AnatomicSite) - } - if (!is.null(specimen$DiseaseStatus)){ - jArray <- convertConceptArray(specimen$DiseaseStatus) - `.jfield<-`(c, 'diseaseStatus', jArray) - } - if (!is.null(specimen$SourceId)){ - sourceId <- convertTextFilter(specimen$SourceId) - `.jfield<-`(c, 'sourceId', sourceId) - } - if (!is.null(specimen$SpecimenSourceConcept)){ - specimenSourceConcept <- .jnew("java/lang/Integer", toString(specimen$SpecimenSourceConcept)) - `.jfield<-`(c, 'specimenSourceConcept', specimenSourceConcept) - } - if (!is.null(specimen$Age)){ - age <- convertNumericRange(specimen$Age) - `.jfield<-`(c, 'age', age) - } - if (!is.null(specimen$Gender)){ - jArray <- convertConceptArray(specimen$Gender) - `.jfield<-`(c, 'gender', jArray) - } - } else if (!is.null(criteria$VisitOccurrence)){ - c = .jnew("org/ohdsi/circe/cohortdefinition/VisitOccurrence") - visitOccurrence <- criteria$VisitOccurrence - if (!is.null(visitOccurrence$CodesetId)){ - codesetId <- .jnew("java/lang/Integer", toString(visitOccurrence$SourceId)) - `.jfield<-`(c, 'codesetId', codesetId) - } - first <- .jnew("java/lang/Boolean", isTRUE(visitOccurrence$First)) - if (!is.null(visitOccurrence$OccurrenceStartDate)){ - occurrenceStartDate <- convertDateRange(visitOccurrence$OccurrenceStartDate) - `.jfield<-`(c, 'occurrenceStartDate', occurrenceStartDate) - } - if (!is.null(visitOccurrence$OccurrenceEndDate)){ - occurrenceEndDate <- convertDateRange(visitOccurrence$OccurrenceEndDate) - `.jfield<-`(c, 'occurrenceEndDate', occurrenceEndDate) - } - if (!is.null(visitOccurrence$VisitType)){ - jArray <- convertConceptArray(visitOccurrence$VisitType) - `.jfield<-`(c, 'visitType', jArray) - } - if (!is.null(visitOccurrence$VisitSourceConcept)){ - visitSourceConcept <- .jnew("java/lang/Integer", toString(visitOccurrence$VisitSourceConcept)) - `.jfield<-`(c, 'visitSourceConcept', visitSourceConcept) - } - if (!is.null(visitOccurrence$VisitLength)){ - visitLength <- convertNumericRange(visitOccurrence$VisitLength) - `.jfield<-`(c, 'visitLength', visitLength) - } - if (!is.null(visitOccurrence$Age)){ - age <- convertNumericRange(visitOccurrence$age) - `.jfield<-`(c, 'age', age) - } - if (!is.null(visitOccurrence$Gender)){ - jArray <- convertConceptArray(visitOccurrence$Gender) - `.jfield<-`(c, 'gender', jArray) - } - if (!is.null(visitOccurrence$ProviderSpecialty)){ - jArray <- convertConceptArray(visitOccurrence$ProviderSpecialty) - `.jfield<-`(c, 'providerSpecialty', jArray) - } - if (!is.null(visitOccurrence$PlaceOfService)){ - jArray <- convertConceptArray(visitOccurrence$PlaceOfService) - `.jfield<-`(c, 'placeOfService', jArray) - } - } - return(c) -} - -convertStrata <- function(strata){ - group <- .jnew("org/ohdsi/circe/cohortdefinition/CriteriaGroup") - `.jfield<-`(group, "type", strata$Type) - count <- .jnew("java/lang/Integer", as.integer(strata$Count)) - `.jfield<-`(group, "count", count) - - # CriteriaList - criteriaList <- list() - for(i in seq_along(strata$CriteriaList)){ - criteria <- strata$CriteriaList[[i]] - cc <- .jnew("org/ohdsi/circe/cohortdefinition/CorelatedCriteria") - - # --- CRITERIA --- - jcc <- convertCriteria(criteria$Criteria) - `.jfield<-`(cc, 'criteria', .jcast(jcc, new.class = "org/ohdsi/circe/cohortdefinition/Criteria")) - - startWindow <- convertWindow(criteria$StartWindow) - `.jfield<-`(cc, 'startWindow', startWindow) - endWindow <- convertWindow(criteria$EndWindow) - `.jfield<-`(cc, 'endWindow', endWindow) - - occurrence <- .jnew("org/ohdsi/circe/cohortdefinition/Occurrence") - type <- as.integer(criteria$Occurrence$Type) - `.jfield<-`(occurrence, 'type', type) - count <- as.integer(criteria$Occurrence$Count) - `.jfield<-`(occurrence, 'count', count) - `.jfield<-`(occurrence, 'isDistinct', isTRUE(criteria$Occurrence$IsDistinct[1])) - `.jfield<-`(cc, 'occurrence', occurrence) - - `.jfield<-`(cc, 'restrictVisit', isTRUE(criteria$RestrictVisit)) - - criteriaList[[i]] <- cc - } - `.jfield<-`(group, 'criteriaList', .jarray(criteriaList, contents.class = "org/ohdsi/circe/cohortdefinition/CorelatedCriteria")) - - # DemographicCriteriaList - demographicCriteria <- list() - for(i in seq_along(strata$DemographicCriteriaList)){ - criteria <- strata$DemographicCriteria[[i]] - dc <- .jnew("org/ohdsi/circe/cohortdefinition/DemographicCriteria") - age <- .jnew("org/ohdsi/circe/cohortdefinition/NumericRange") - `.jfield<-`(age, 'value', criteria$Age$Value) - `.jfield<-`(age, 'op', criteria$Age$Op) - `.jfield<-`(age, 'extent', criteria$Age$Extent) - `.jfield<-`(dc, 'age', age) - `.jfield<-`(dc, 'gender', convertConceptArray(criteria$Gender)) - `.jfield<-`(dc, 'race', convertConceptArray(criteria$Race)) - `.jfield<-`(dc, 'ethnicity', convertConceptArray(criteria$Ethnicity)) - `.jfield<-`(dc, 'occurenceStartDate', convertDateRange(criteria$OccurenceStartDate)) - `.jfield<-`(dc, 'occurenceEndDate', convertDateRange(criteria$OccurenceEndDate)) - demographicCriteria[[i]] <- dc - } - `.jfield<-`(group, 'demographicCriteriaList', .jarray(demographicCriteria, contents.class = "org/ohdsi/circe/cohortdefinition/DemographicCriteria")) - - # Groups - groups <- list() - for(i in seq_along(strata$Groups)){ - gr <- strata$Groups[[i]] - g <- convertStrata(gr) - groups[[i]] <- g - } - `.jfield<-`(group, 'groups', .jarray(groups, contents.class = "org/ohdsi/circe/cohortdefinition/CriteriaGroup")) - - return(group); -} - -getStrataQuery <- function(strataCriteria, dbms){ - - builder <- .jnew("org/ohdsi/circe/cohortdefinition/CohortExpressionQueryBuilder") - jStrataCriteria <- convertStrata(strataCriteria) - tryCatch(criteria <- .jcall(builder, returnSig = 'S', 'getCriteriaGroupQuery', jStrataCriteria, "#analysis_events"), - NullPointerException = function(e){ - print(e) - e$jobj$printStackTrace() - stop() - }) - additionalCriteriaQuery <- paste("\nJOIN (\n", criteria, ") AC on AC.person_id = pe.person_id AND AC.event_id = pe.event_id") - indexId <- 0 - sql <- SqlRender::readSql("strata.sql") - sql <- SqlRender::renderSql(sql, - additionalCriteriaQuery = gsub("@indexId", "0", additionalCriteriaQuery), - indexId = indexId)$sql - sql <- SqlRender::translateSql(sql, targetDialect = dbms)$sql - return (sql) -} - -convertExpression <- function(expression){ - cse <- .jnew("org/ohdsi/circe/vocabulary/ConceptSetExpression") - items <- list() - for(i in seq_along(expression$items)){ - expr <- expression$items[[i]] - item <- .jnew("org/ohdsi/circe/vocabulary/ConceptSetExpression$ConceptSetItem") - concept <- convertConcept(expr$concept) - `.jfield<-`(item, 'concept', concept) - `.jfield<-`(item, 'isExcluded', isTRUE(expr$isExcluded)) - `.jfield<-`(item, 'includeDescendants', isTRUE(expr$includeDescendants)) - `.jfield<-`(item, 'includeMapped', isTRUE(expr$includeMapped)) - items[[i]] <- item - } - `.jfield<-`(cse, 'items', .jarray(items, contents.class = "org/ohdsi/circe/vocabulary/ConceptSetExpression$ConceptSetItem")) - return(cse) -} - -convertConceptSet <- function(conceptSet){ - cs <- .jnew("org/ohdsi/circe/cohortdefinition/ConceptSet") - `.jfield<-`(cs, 'id', as.integer(conceptSet$id)) - `.jfield<-`(cs, 'name', conceptSet$name) - expr <- convertExpression(conceptSet$expression) - `.jfield<-`(cs, 'expression', expr) - return(cs) -} - -convertConceptSetArray <- function(conceptSets){ - cs <- list() - for(i in seq_along(conceptSets)){ - conceptSet <- conceptSets[[i]] - cs[[i]] <- convertConceptSet(conceptSet) - } - return(.jarray(cs, contents.class = "org/ohdsi/circe/cohortdefinition/ConceptSet")) -} - -getCodesetQuery <- function(conceptSets){ - jInit = NULL - builder <- .jnew("org/ohdsi/circe/cohortdefinition/CohortExpressionQueryBuilder", jInit) - arg <- convertConceptSetArray(conceptSets) - sql <- .jcall(builder, returnSig = "S", 'getCodesetQuery', arg) - return(sql) -} - -buildAnalysisQuery <- function(analysisExpression, analysisId, dbms, cdmSchema, resultsDatabaseSchema){ - - cohortIdStatements <- list() - for(i in seq_along(analysisExpression$targetIds)){ - id <- analysisExpression$targetIds[[i]] - stmt <- paste("SELECT ", id, " as cohort_id, 0 as is_outcome") - cohortIdStatements[[i]] <- stmt - } - outcomeIdStatements <- list() - for(i in seq_along(analysisExpression$outcomeIds)){ - id <- analysisExpression$outcomeIds[[i]] - stmt <- paste("SELECT ", id, " as cohort_id, 1 as is_outcome") - outcomeIdStatements[[i]] <- stmt - } - targets <- paste(cohortIdStatements, collapse = " UNION ") - outcomes <- paste(outcomeIdStatements, collapse = " UNION ") - cohortInserts <- paste(targets, " UNION ", outcomes) - write(paste("Cohort inserts: ", cohortInserts), stdout()) - - dateField <- analysisExpression$timeAtRisk$start$DateField - if (!is.null(dateField) && "StartDate" == dateField) { - startDay <- "cohort_start_date" - } else { - startDay <- "cohort_end_date" - } - adjustedStart <- paste("DATEADD(day,", analysisExpression$timeAtRisk$start$Offset, ",", startDay, ")") - dateField <- analysisExpression$timeAtRisk$end$DateField - if (!is.null(dateField) && dateField == "StartDate") { - endDay <- "cohort_start_date" - } else { - endDay <- "cohort_end_date" - } - adjustedEnd <- paste("DATEADD(day,", analysisExpression$timeAtRisk$end$Offset, ",", endDay, ")") - - studyWindowClauses <- list() - if (!is.null(analysisExpression$studyWindow)){ - i <- 1 - if (!is.null(analysisExpression$studyWindow$startDate) && length(analysisExpression$studyWindow$startDate) > 0){ - studyWindowClauses[[i]] <- paste("t.cohort_start_date >= '", analysisExpression$studyWindow$startDate, "'", collapse = "") - i <- i + 1 - } - if (!is.null(analysisExpression$studyWindow$endDate) && length(analysisExpression$studyWindow$endDate) > 0){ - studyWindowClauses[[i]] <- paste("t.cohort_start_date <= '", analysisExpression$studyWindow$endDate, "'", collapse = "") - } - } - cohortDataFilter <- "" - if (length(studyWindowClauses) > 0){ - cohortDataFilter <- paste("AND ", paste(studyWindowClauses, collapse = " AND ")) - } - - endDateUnions <- "" - if (!is.null(analysisExpression$studyWindow) && !is.null(analysisExpression$studyWindow$endDate) && length(analysisExpression$studyWindow$endDate) > 0){ - endDateUnions <- paste("UNION\nselect combos.target_id, combos.outcome_id, t.subject_id, t.cohort_start_date, '", analysisExpression$studyWindow$endDate, "' as followup_end, 0 as is_case\n FROM cteCohortCombos combos\n JOIN cteCohortData t on combos.target_id = t.target_id and combos.outcome_id = t.outcome_id") - } - - codesetQuery = getCodesetQuery(analysisExpression$ConceptSets) -# write(paste("Codeset Query: ", codesetQuery), stdout()) - - strataInsert <- list() - for(i in seq_along(analysisExpression$strata)){ - strata <- analysisExpression$strata[[i]] - cg <- strata[[1]]$expression - st <- getStrataQuery(cg, dbms) - stratumInsert <- gsub("@strata_sequence", i, st) - strataInsert[[i]] <- stratumInsert - } - strataCohortInserts <- paste(strataInsert, collapse = "\n") -# write(paste("Strata Cohort Inserts: ", strataCohortInserts), stdout()) - - sql <- SqlRender::readSql("performAnalysis.sql") - sql <- gsub("@cohortInserts", cohortInserts, sql) - sql <- gsub("@strataCohortInserts", strataCohortInserts, sql) - sql <- gsub("@cohortDataFilter", cohortDataFilter, sql) - sql <- gsub("@codesetQuery", codesetQuery, sql) - sql <- gsub("@EndDateUnions", endDateUnions, sql) - sql <- SqlRender::renderSql(sql, - results_database_schema = resultsDatabaseSchema, - adjustedStart = adjustedStart, - adjustedEnd = adjustedEnd, - cdm_database_schema = cdmSchema, - results_database_schema = resultsDatabaseSchema)$sql - sql = gsub("@cdm_database_schema", cdmSchema, sql) - sql = gsub("@results_database_schema", resultsDatabaseSchema, sql) - sql = gsub("@analysisId", toString(analysisId), sql) -# sql <- SqlRender::translateSql(sql, targetDialect = dbms)$sql - return(sql) -} \ No newline at end of file diff --git a/src/main/resources/r/ir/ir_dist.sql b/src/main/resources/r/ir/ir_dist.sql deleted file mode 100644 index 7cf0cc0b5..000000000 --- a/src/main/resources/r/ir/ir_dist.sql +++ /dev/null @@ -1,19 +0,0 @@ -select - target_id, - outcome_id, - strata_sequence, - dist_type, - total, - avg_value, - std_dev, - min_value, - p10_value, - p25_value, - median_value, - p75_value, - p90_value, - max_value -from - @resultsSchema.ir_analysis_dist -where - analysis_id = @analysisId \ No newline at end of file diff --git a/src/main/resources/r/ir/performAnalysis.sql b/src/main/resources/r/ir/performAnalysis.sql deleted file mode 100644 index f9ef3b39d..000000000 --- a/src/main/resources/r/ir/performAnalysis.sql +++ /dev/null @@ -1,316 +0,0 @@ -select cohort_id, is_outcome -into #cohorts -FROM ( - @cohortInserts -) C -; - -with cteCohortCombos (target_id, outcome_id) as -( - select t.cohort_id as target_id, o.cohort_id as outcome_id - FROM #cohorts t - CROSS JOIN #cohorts o - where t.is_outcome = 0 and o.is_outcome = 1 -), -cteCohortData(target_id, outcome_id, subject_id, cohort_start_date, cohort_end_date, adjusted_start_date, adjusted_end_date, op_start_date, op_end_date) as -( - select combos.target_id, combos.outcome_id, t.subject_id, t.cohort_start_date, t.cohort_end_date, t.adjusted_start_date, t.adjusted_end_date, op.observation_period_start_date as op_start_date, op.observation_period_end_date as op_end_date - from cteCohortCombos combos - join ( - select cohort_definition_id, subject_id, cohort_start_date, cohort_end_date, @adjustedStart as adjusted_start_date, @adjustedEnd as adjusted_end_date - FROM @results_database_schema.cohort - ) t on t.cohort_definition_id = combos.target_id - join @cdm_database_schema.observation_period op on t.subject_id = op.person_id and t.cohort_start_date between op.observation_period_start_date and op.observation_period_end_date - left join ( - select cohort_definition_id, subject_id, min(cohort_start_date) as cohort_start_date - from @results_database_schema.cohort - GROUP BY cohort_definition_id, subject_id - ) O on o.cohort_definition_id = combos.outcome_id - and t.subject_id = o.subject_id - where (o.cohort_start_date is null or o.cohort_start_date > t.adjusted_start_date) - and t.adjusted_start_date < t.adjusted_end_date - and t.adjusted_start_date between op.observation_period_start_date and op.observation_period_end_date - @cohortDataFilter -), -cteEndDates (target_id, outcome_id, subject_id, cohort_start_date, followup_end, is_case) as -( - select target_id, outcome_id, subject_id, cohort_start_date, followup_end, is_case - FROM ( - select target_id, outcome_id, subject_id, cohort_start_date, followup_end, is_case, row_number() over (partition by target_id, outcome_id, subject_id, cohort_start_date order by followup_end asc, is_case desc) as RN - FROM ( - select combos.target_id, combos.outcome_id, t.subject_id, t.cohort_start_date, t.op_end_date as followup_end, 0 as is_case - from cteCohortCombos combos - join cteCohortData t on combos.target_id = t.target_id and combos.outcome_id = t.outcome_id - - UNION - select combos.target_id, combos.outcome_id, t.subject_id, t.cohort_start_date, t.adjusted_end_date as followup_end, 0 as is_case - from cteCohortCombos combos - join cteCohortData t on combos.target_id = t.target_id and combos.outcome_id = t.outcome_id - - UNION - select combos.target_id, combos.outcome_id, t.subject_id, t.cohort_start_date, o.cohort_start_date as followup_end, 1 as is_case - from cteCohortCombos combos - join cteCohortData t on combos.target_id = t.target_id and combos.outcome_id = t.outcome_id - join ( - select cohort_definition_id, subject_id, min(cohort_start_date) as cohort_start_date - from @results_database_schema.cohort - GROUP BY cohort_definition_id, subject_id - ) O on o.cohort_definition_id = combos.outcome_id and t.subject_id = o.subject_id - where o.cohort_start_date > t.adjusted_start_date - - @EndDateUnions - - ) RawData - ) Result - WHERE Result.RN = 1 -), -cteRawData (target_id, outcome_id, subject_id, cohort_start_date, cohort_end_date, time_at_risk, is_case) as -( - select t.target_id, t.outcome_id, t.subject_id, t.cohort_start_date, t.cohort_end_date, datediff(d,t.adjusted_start_date, e.followup_end) as time_at_risk, e.is_case - from cteCohortData t - join cteEndDates e on t.target_id = e.target_id - and t.outcome_id = e.outcome_id - and t.subject_id = e.subject_id - and t.cohort_start_date = e.cohort_start_date -) -select target_id, outcome_id, subject_id, cohort_start_date, cohort_end_date, time_at_risk, is_case -INTO #time_at_risk -from cteRawData -; - --- from here, take all the people's person_id, start_date, end_date, create an 'events table' -select row_number() over (partition by P.person_id order by P.start_date) as event_id, P.person_id, P.start_date, P.end_date, P.op_start_date, P.op_end_date -INTO #analysis_events -FROM -( - select distinct T.subject_id as person_id, T.cohort_start_date as start_date, T.cohort_end_date as end_date, OP.observation_period_start_date as op_start_date, OP.observation_period_end_date as op_end_date - from #time_at_risk T - JOIN @cdm_database_schema.observation_period OP on T.subject_id = OP.person_id and T.cohort_start_date between OP.observation_period_start_date and OP.observation_period_end_date -) P -; - --- create the stratifiction set -@codesetQuery - -create table #strataCohorts -( - strata_sequence int, - person_id bigint, - event_id bigint -) -; -@strataCohortInserts - --- join back the followup to the stratification and write to the results page. -DELETE FROM @results_database_schema.ir_analysis_result where analysis_id = @analysisId; - -INSERT INTO @results_database_schema.ir_analysis_result (analysis_id, target_id, outcome_id, strata_mask, person_count, time_at_risk, cases) -select @analysisId as analysis_id, T.target_id, T.outcome_id, E.strata_mask, - COUNT(subject_id) as person_count, - sum(1.0 * time_at_risk / 365.25) as time_at_risk, - sum(is_case) as cases -from #time_at_risk T -JOIN ( - select E.event_id, E.person_id, E.start_date, E.end_date, SUM(coalesce(POWER(cast(2 as bigint), SC.strata_sequence), 0)) as strata_mask - FROM #analysis_events E - LEFT JOIN #strataCohorts SC on SC.person_id = E.person_id and SC.event_id = E.event_id - group by E.event_id, E.person_id, E.start_date, E.end_date -) E on T.subject_id = E.person_id and T.cohort_start_date = E.start_date and T.cohort_end_date = E.end_date -GROUP BY T.target_id, T.outcome_id, E.strata_mask -; --- note in the case of no stratification (no rows in strataCohorts temp table), everyone will have strata_mask of 0. - --- calculate the individual strata counts from the raw person data. Rows from #strataCohorts are used to find counts for each strata -delete from @results_database_schema.ir_analysis_strata_stats where analysis_id = @analysisId; -insert into @results_database_schema.ir_analysis_strata_stats (analysis_id, target_id, outcome_id, strata_sequence, person_count, time_at_risk, cases) -select irs.analysis_id, combos.target_id, combos.outcome_id, irs.strata_sequence, coalesce(T.person_count, 0) as person_count, coalesce(T.time_at_risk, 0) as time_at_risk, coalesce(T.cases, 0) as cases -from @results_database_schema.ir_strata irs -cross join ( - select t.cohort_id as target_id, o.cohort_id as outcome_id - FROM #cohorts t - CROSS JOIN #cohorts o - where t.is_outcome = 0 and o.is_outcome = 1 -) combos -left join -( - select T.target_id, T.outcome_id, S.strata_sequence, count(S.event_id) as person_count, sum(1.0 * T.time_at_risk / 365.25) as time_at_risk, sum(T.is_case) as cases - from #analysis_events E - JOIN #strataCohorts S on S.person_id = E.person_id and E.event_id = S.event_id - join #time_at_risk T on T.subject_id = E.person_id and T.cohort_start_date = E.start_date and T.cohort_end_date = E.end_date - group by T.target_id, T.outcome_id, S.strata_sequence -) T on irs.strata_sequence = T.strata_sequence and T.target_id = combos.target_id and T.outcome_id = combos.outcome_id -WHERE irs.analysis_id = @analysisId -; - --- calculate distributions for TAR and TTO by strata - -DELETE FROM @results_database_schema.ir_analysis_dist where analysis_id = @analysisId; - --- dist_type 1: time at risk - -WITH cteRawData (target_id, outcome_id, strata_sequence, count_value) as -( - select T.target_id, T.outcome_id, -1 as strata_sequence, T.time_at_risk as count_value - from #time_at_risk T - - UNION ALL - - select T.target_id, T.outcome_id, S.strata_sequence, T.time_at_risk as count_value - from #analysis_events E - JOIN #strataCohorts S on E.person_id = S.person_id and E.event_id = S.event_id - join #time_at_risk T on T.subject_id = E.person_id and T.cohort_start_date = E.start_date and T.cohort_end_date = E.end_date -) -, overallStats (target_id, outcome_id, strata_sequence, avg_value, stdev_value, min_value, max_value, total) as -( - select target_id, - outcome_id, - strata_sequence, - avg(1.0 * count_value) as avg_value, - stdev(count_value) as stdev_value, - min(count_value) as min_value, - max(count_value) as max_value, - count_big(*) as total - from cteRawData - group by target_id, outcome_id, strata_sequence -), -stats (target_id, outcome_id, strata_sequence, count_value, total, rn) as -( - select target_id, outcome_id, strata_sequence, count_value, count_big(*) as total, row_number() over (partition by target_id, outcome_id, strata_sequence order by count_value) as rn - FROM cteRawData - group by target_id, outcome_id, strata_sequence, count_value -), -priorStats (target_id, outcome_id, strata_sequence, count_value, total, accumulated) as -( - select s.target_id, s.outcome_id, s.strata_sequence, s.count_value, s.total, sum(p.total) as accumulated - from stats s - join stats p on s.target_id = p.target_id and s.outcome_id = p.outcome_id and s.strata_sequence = p.strata_sequence and p.rn <= s.rn - group by s.target_id, s.outcome_id, s.strata_sequence, s.count_value, s.total, s.rn -) -select - o.target_id, - o.outcome_id, - o.strata_sequence, - o.total, - o.avg_value, - coalesce(o.stdev_value, 0.0) as stdev_value, - o.min_value, - MIN(case when p.accumulated >= .10 * o.total then count_value else o.max_value end) as p10_value, - MIN(case when p.accumulated >= .25 * o.total then count_value else o.max_value end) as p25_value, - MIN(case when p.accumulated >= .50 * o.total then count_value else o.max_value end) as median_value, - MIN(case when p.accumulated >= .75 * o.total then count_value else o.max_value end) as p75_value, - MIN(case when p.accumulated >= .90 * o.total then count_value else o.max_value end) as p90_value, - o.max_value -INTO #tempTARDist -from priorStats p -join overallStats o on p.target_id = o.target_id and p.outcome_id = o.outcome_id and p.strata_sequence = o.strata_sequence -GROUP BY o.target_id, o.outcome_id, o.strata_sequence, o.total, o.min_value, o.max_value, o.avg_value, o.stdev_value -; - -INSERT INTO @results_database_schema.ir_analysis_dist (analysis_id, dist_type, target_id, outcome_id, strata_sequence, total, avg_value, std_dev,min_value, p10_value, p25_value, median_value, p75_value, p90_value,max_value) -select @analysisId as analysis_id, 1 as dist_type, combos.target_id, combos.outcome_id, - case when d.strata_sequence = -1 then null else d.strata_sequence end as strata_sequence, - d.total, d.avg_value, d.stdev_value, d.min_value, d.p10_value, d.p25_value, d.median_value, d.p75_value, d.p90_value, d.max_value -FROM -( - select t.cohort_id as target_id, o.cohort_id as outcome_id - FROM #cohorts t - CROSS JOIN #cohorts o - where t.is_outcome = 0 and o.is_outcome = 1 -) combos -JOIN #tempTARDist d on combos.target_id = d.target_id and combos.outcome_id = d.outcome_id -; - -TRUNCATE TABLE #tempTARDist; -DROP TABLE #tempTARDist; - --- dist_type 2: TTO (time to outcome) - -WITH cteRawData (target_id, outcome_id, strata_sequence, count_value) as -( - select T.target_id, T.outcome_id, -1 as strata_sequence, T.time_at_risk as count_value - from #time_at_risk T - where T.is_case = 1 - - UNION ALL - - select T.target_id, T.outcome_id, S.strata_sequence, T.time_at_risk as count_value - from #analysis_events E - JOIN #strataCohorts S on E.person_id = S.person_id and E.event_id = S.event_id - join #time_at_risk T on T.subject_id = E.person_id and T.cohort_start_date = E.start_date and T.cohort_end_date = E.end_date - where T.is_case = 1 -) -, overallStats (target_id, outcome_id, strata_sequence, avg_value, stdev_value, min_value, max_value, total) as -( - select target_id, - outcome_id, - strata_sequence, - avg(1.0 * count_value) as avg_value, - stdev(count_value) as stdev_value, - min(count_value) as min_value, - max(count_value) as max_value, - count_big(*) as total - from cteRawData - group by target_id, outcome_id, strata_sequence -), -stats (target_id, outcome_id, strata_sequence, count_value, total, rn) as -( - select target_id, outcome_id, strata_sequence, count_value, count_big(*) as total, row_number() over (partition by target_id, outcome_id, strata_sequence order by count_value) as rn - FROM cteRawData - group by target_id, outcome_id, strata_sequence, count_value -), -priorStats (target_id, outcome_id, strata_sequence, count_value, total, accumulated) as -( - select s.target_id, s.outcome_id, s.strata_sequence, s.count_value, s.total, sum(p.total) as accumulated - from stats s - join stats p on s.target_id = p.target_id and s.outcome_id = p.outcome_id and s.strata_sequence = p.strata_sequence and p.rn <= s.rn - group by s.target_id, s.outcome_id, s.strata_sequence, s.count_value, s.total, s.rn -) -select - o.target_id, - o.outcome_id, - o.strata_sequence, - o.total, - o.avg_value, - coalesce(o.stdev_value, 0.0) as stdev_value, - o.min_value, - MIN(case when p.accumulated >= .10 * o.total then count_value else o.max_value end) as p10_value, - MIN(case when p.accumulated >= .25 * o.total then count_value else o.max_value end) as p25_value, - MIN(case when p.accumulated >= .50 * o.total then count_value else o.max_value end) as median_value, - MIN(case when p.accumulated >= .75 * o.total then count_value else o.max_value end) as p75_value, - MIN(case when p.accumulated >= .90 * o.total then count_value else o.max_value end) as p90_value, - o.max_value -INTO #tempTTODist -from priorStats p -join overallStats o on p.target_id = o.target_id and p.outcome_id = o.outcome_id and p.strata_sequence = o.strata_sequence -GROUP BY o.target_id, o.outcome_id, o.strata_sequence, o.total, o.min_value, o.max_value, o.avg_value, o.stdev_value -; - -INSERT INTO @results_database_schema.ir_analysis_dist (analysis_id, dist_type, target_id, outcome_id, strata_sequence, total, avg_value, std_dev,min_value, p10_value, p25_value, median_value, p75_value, p90_value,max_value) -select @analysisId as analysis_id, 2 as dist_type, combos.target_id, combos.outcome_id, - case when d.strata_sequence = -1 then null else d.strata_sequence end as strata_sequence, - d.total, d.avg_value, d.stdev_value, d.min_value, d.p10_value, d.p25_value, d.median_value, d.p75_value, d.p90_value, d.max_value -FROM -( - select t.cohort_id as target_id, o.cohort_id as outcome_id - FROM #cohorts t - CROSS JOIN #cohorts o - where t.is_outcome = 0 and o.is_outcome = 1 -) combos -JOIN #tempTTODist d on combos.target_id = d.target_id and combos.outcome_id = d.outcome_id -; - -TRUNCATE TABLE #tempTTODist; -DROP TABLE #tempTTODist; - -TRUNCATE TABLE #Codesets; -DROP TABLE #Codesets; - -TRUNCATE TABLE #strataCohorts; -DROP TABLE #strataCohorts; - -TRUNCATE TABLE #cohorts; -DROP TABLE #cohorts; - -TRUNCATE TABLE #time_at_risk; -DROP TABLE #time_at_risk; - diff --git a/src/main/resources/r/ir/strata.sql b/src/main/resources/r/ir/strata.sql deleted file mode 100644 index 14fa4deb3..000000000 --- a/src/main/resources/r/ir/strata.sql +++ /dev/null @@ -1,9 +0,0 @@ -INSERT INTO #strataCohorts (strata_sequence, person_id, event_id) -select @strata_sequence as strata_id, person_id, event_id -FROM -( - select pe.person_id, pe.event_id - FROM #analysis_events pe - @additionalCriteriaQuery -) Results -; diff --git a/src/main/resources/r/ir/strata_rules.sql b/src/main/resources/r/ir/strata_rules.sql deleted file mode 100644 index 0a3353038..000000000 --- a/src/main/resources/r/ir/strata_rules.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO @results_schema.ir_strata (analysis_id, strata_sequence, name, description) VALUES (@analysis_id,@strata_sequence,'@name','@description') \ No newline at end of file diff --git a/src/main/resources/r/ir/strata_stats.sql b/src/main/resources/r/ir/strata_stats.sql deleted file mode 100644 index db5c71178..000000000 --- a/src/main/resources/r/ir/strata_stats.sql +++ /dev/null @@ -1,5 +0,0 @@ -select r.analysis_id, r.target_id, r.outcome_id, r.strata_sequence, s.name, sum(person_count) as person_count, sum(time_at_risk) as time_at_risk, sum(cases) as cases -from @results_database_schema.ir_analysis_strata_stats r -join @results_database_schema.ir_strata s on r.analysis_id = s.analysis_id and r.strata_sequence = s.strata_sequence -where r.analysis_id = @analysis_id -GROUP BY r.analysis_id, r.target_id, r.outcome_id, r.strata_sequence, s.name diff --git a/src/test/java/com/odysseusinc/arachne/portal/TestApplication.java b/src/test/java/com/odysseusinc/arachne/portal/TestApplication.java index d72f7d466..2a1d5be2b 100644 --- a/src/test/java/com/odysseusinc/arachne/portal/TestApplication.java +++ b/src/test/java/com/odysseusinc/arachne/portal/TestApplication.java @@ -164,9 +164,7 @@ protected void configure(HttpSecurity http) throws Exception { @Bean public AuthenticationTokenFilter authenticationTokenFilterBean() throws Exception { - AuthenticationTokenFilter authenticationTokenFilter = new AuthenticationTokenFilter(); - authenticationTokenFilter.setAuthenticationManager(authenticationManagerBean()); - return authenticationTokenFilter; + return new AuthenticationTokenFilter(); } @Autowired diff --git a/src/test/java/com/odysseusinc/arachne/portal/service/submission/impl/SubmissionServiceImplTest.java b/src/test/java/com/odysseusinc/arachne/portal/service/submission/impl/SubmissionServiceImplTest.java new file mode 100644 index 000000000..3a499ba66 --- /dev/null +++ b/src/test/java/com/odysseusinc/arachne/portal/service/submission/impl/SubmissionServiceImplTest.java @@ -0,0 +1,104 @@ +package com.odysseusinc.arachne.portal.service.submission.impl; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.odysseusinc.arachne.portal.repository.ResultFileRepository; +import com.odysseusinc.arachne.portal.repository.SubmissionFileRepository; +import com.odysseusinc.arachne.portal.repository.SubmissionGroupRepository; +import com.odysseusinc.arachne.portal.repository.SubmissionInsightRepository; +import com.odysseusinc.arachne.portal.repository.SubmissionResultFileRepository; +import com.odysseusinc.arachne.portal.repository.SubmissionStatusHistoryRepository; +import com.odysseusinc.arachne.portal.repository.submission.BaseSubmissionRepository; +import com.odysseusinc.arachne.portal.service.BaseDataSourceService; +import com.odysseusinc.arachne.portal.service.UserService; +import com.odysseusinc.arachne.portal.service.mail.ArachneMailSender; +import com.odysseusinc.arachne.portal.util.AnalysisHelper; +import com.odysseusinc.arachne.portal.util.ContentStorageHelper; +import com.odysseusinc.arachne.portal.util.LegacyAnalysisHelper; +import com.odysseusinc.arachne.portal.util.SubmissionHelper; +import com.odysseusinc.arachne.storage.service.ContentStorageService; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import javax.persistence.EntityManager; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.messaging.simp.SimpMessagingTemplate; + +@RunWith(MockitoJUnitRunner.class) +public class SubmissionServiceImplTest { + + @InjectMocks + @Spy + private SubmissionServiceImpl submissionService; + + @Mock + private BaseSubmissionRepository submissionRepository; + @Mock + private BaseDataSourceService dataSourceService; + @Mock + private ArachneMailSender mailSender; + @Mock + private AnalysisHelper analysisHelper; + @Mock + private SimpMessagingTemplate wsTemplate; + @Mock + private LegacyAnalysisHelper legacyAnalysisHelper; + @Mock + private SubmissionResultFileRepository submissionResultFileRepository; + @Mock + private SubmissionGroupRepository submissionGroupRepository; + @Mock + private SubmissionInsightRepository submissionInsightRepository; + @Mock + private SubmissionFileRepository submissionFileRepository; + @Mock + private ResultFileRepository resultFileRepository; + @Mock + private SubmissionStatusHistoryRepository submissionStatusHistoryRepository; + @Mock + private EntityManager entityManager; + @Mock + private SubmissionHelper submissionHelper; + @Mock + private ContentStorageService contentStorageService; + @Mock + private UserService userService; + @Mock + private ContentStorageHelper contentStorageHelper; + + @Captor + private ArgumentCaptor fileNamesCaptor; + + + + @Test + public void uploadResultsByDataOwner_zipFile() throws IOException { + + doReturn(null) + .when(submissionService) + .uploadResultsByDataOwner(any(), any(), any(File.class)); + + URL zipFileUrl = getClass().getClassLoader().getResource("submission/test.zip"); + + submissionService.uploadResultsByDataOwner(1L, new File(zipFileUrl.getPath())); + + verify(submissionService, times(2)) + .uploadResultsByDataOwner(any(), fileNamesCaptor.capture(), any(File.class)); + + assertEquals(Arrays.asList("test2.txt", "test3.txt"), fileNamesCaptor.getAllValues()); + + } + +} \ No newline at end of file diff --git a/src/test/resources-binary/submission/test.zip b/src/test/resources-binary/submission/test.zip new file mode 100644 index 000000000..2d08a0225 Binary files /dev/null and b/src/test/resources-binary/submission/test.zip differ diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 7ce0abb3e..f125791c6 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -28,7 +28,7 @@ spring.datasource.password=ohdsi spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.max-active=400 spring.datasource.connection-test-query=select 1 -arachne.token.secret=sssshhhh! +arachne.token.secret=129DF19C8A91AFD8375A2826A33539K01ACQ778QOJFAA9MGWLWH73PLXVFVHBR7860MTIE2O8EEVF9KCO77P6A7NUNX4XHAGCRFSBWG879XPDOIN6C2LFCKJI002OIABS4D6Q9VMJJIX8UCE48EF arachne.token.header=Arachne-Auth-Token arachne.systemToken.header=Arachne-System-Token arachne.token.expiration=604800 @@ -95,4 +95,16 @@ spring.activemq.packages.trust-all=true jodconverter.enabled=false antivirus.host=localhost -antivirus.port=3310 \ No newline at end of file +antivirus.port=3310 +authenticator.methods.db.service=org.ohdsi.authenticator.service.jdbc.JdbcAuthService +authenticator.methods.db.config.jdbcUrl=${spring.datasource.url} +authenticator.methods.db.config.username=${spring.datasource.username} +authenticator.methods.db.config.password=${spring.datasource.password} +authenticator.methods.db.config.query=SELECT password, firstname, middlename, lastname FROM users_data WHERE email = :username +authenticator.methods.db.config.passwordEncoder=org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder +authenticator.methods.db.config.fieldsToExtract.firstName=firstname +authenticator.methods.db.config.fieldsToExtract.middleName=middlename +authenticator.methods.db.config.fieldsToExtract.lastName=lastname +security.method=db +security.jwt.token.secretKey=${arachne.token.secret} +security.jwt.token.validityInSeconds=${arachne.token.expiration} \ No newline at end of file