Merge pull request #9 from bento-platform/releases/v2.3.0

releases/v2.3.0

src/api/main.go

@@ -122,27 +122,33 @@ func main() {
 		// middleware
 		gam.MandateChromosomeAttribute,
 		gam.MandateCalibratedBounds,
+		gam.MandateAssemblyIdAttribute,
 		gam.ValidatePotentialGenotypeQueryParameter)
 	e.GET("/variants/get/by/sampleId", mvc.VariantsGetBySampleId,
 		// middleware
 		gam.MandateChromosomeAttribute,
 		gam.MandateCalibratedBounds,
+		gam.MandateAssemblyIdAttribute,
 		gam.MandateSampleIdsPluralAttribute,
 		gam.ValidatePotentialGenotypeQueryParameter)
 
 	e.GET("/variants/count/by/variantId", mvc.VariantsCountByVariantId,
 		// middleware
 		gam.MandateChromosomeAttribute,
 		gam.MandateCalibratedBounds,
+		gam.MandateAssemblyIdAttribute,
 		gam.ValidatePotentialGenotypeQueryParameter)
 	e.GET("/variants/count/by/sampleId", mvc.VariantsCountBySampleId,
 		// middleware
 		gam.MandateChromosomeAttribute,
 		gam.MandateCalibratedBounds,
+		gam.MandateAssemblyIdAttribute,
 		gam.MandateSampleIdsSingularAttribute,
 		gam.ValidatePotentialGenotypeQueryParameter)
 
-	e.GET("/variants/ingestion/run", mvc.VariantsIngest)
+	e.GET("/variants/ingestion/run", mvc.VariantsIngest,
+		// middleware
+		gam.MandateAssemblyIdAttribute)
 	e.GET("/variants/ingestion/requests", mvc.GetAllVariantIngestionRequests)
 
 	// Run

src/api/middleware/assemblyMiddleware.go

@@ -0,0 +1,24 @@
+package middleware
+
+import (
+	assid "api/models/constants/assembly-id"
+	"net/http"
+
+	"github.com/labstack/echo"
+)
+
+/*
+	Echo middleware to ensure a valid `assemblyId` HTTP query parameter was provided
+*/
+func MandateAssemblyIdAttribute(next echo.HandlerFunc) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		// check for assemblyId query parameter
+		assemblyId := c.QueryParam("assemblyId")
+		if len(assemblyId) == 0 || !assid.IsKnownAssemblyId(assemblyId) {
+			// if no id was provided, or it was invalid, return an error
+			return echo.NewHTTPError(http.StatusBadRequest, "Missing or unknown assemblyId!")
+		}
+
+		return next(c)
+	}
+}

src/api/models/constants/assembly-id/main.go

@@ -0,0 +1,36 @@
+package assemblyId
+
+import (
+	"api/models/constants"
+	"strings"
+)
+
+const (
+	Unknown constants.AssemblyId = "Unknown"
+
+	GRCh38 constants.AssemblyId = "GRCh38"
+	GRCh37 constants.AssemblyId = "GRCh37"
+	NCBI36 constants.AssemblyId = "NCBI36"
+	Other  constants.AssemblyId = "Other"
+)
+
+func CastToAssemblyId(text string) constants.AssemblyId {
+	switch strings.ToLower(text) {
+	case "grch38":
+		return GRCh38
+	case "grch37":
+		return GRCh37
+	case "ncbi36":
+		return NCBI36
+	case "other":
+		return Other
+	default:
+		return Unknown
+	}
+}
+
+func IsKnownAssemblyId(text string) bool {
+	// attempt to cast to assemblyId and
+	// return if unknown assemblyId
+	return CastToAssemblyId(text) != Unknown
+}

src/api/models/constants/main.go

@@ -6,6 +6,7 @@ package constants
 	throughout Gohan and it's
 	associated services.
 */
+type AssemblyId string
 type GenotypeQuery string
 type SearchOperation string
 type SortDirection string

src/api/models/elasticsearch.go

@@ -18,7 +18,9 @@ type Variant struct {
 	Info   []Info   `json:"info"`
 
 	Samples []Sample `json:"samples"`
-	FileId  string   `json:"fileId"`
+
+	FileId     string       `json:"fileId"`
+	AssemblyId c.AssemblyId `json:"assemblyId"`
 }
 
 type Info struct {

src/api/mvc/variants.go

@@ -16,6 +16,7 @@ import (
 	"api/contexts"
 	"api/models"
 	"api/models/constants"
+	a "api/models/constants/assembly-id"
 	gq "api/models/constants/genotype-query"
 	s "api/models/constants/sort"
 	"api/models/ingest"
@@ -93,6 +94,8 @@ func VariantsIngest(c echo.Context) error {
 		}
 	}
 
+	assemblyId := a.CastToAssemblyId(c.QueryParam("assemblyId"))
+
 	startTime := time.Now()
 
 	fmt.Printf("Ingest Start: %s\n", startTime)
@@ -214,7 +217,7 @@ func VariantsIngest(c echo.Context) error {
 			}
 
 			// ---	 load back into memory and process
-			ingestionService.ProcessVcf(vcfFilePath, drsFileId)
+			ingestionService.ProcessVcf(vcfFilePath, drsFileId, assemblyId)
 
 			// ---   delete the temporary vcf file
 			os.Remove(vcfFilePath)
@@ -287,6 +290,10 @@ func GetVariantsOverview(c echo.Context) error {
 	wg.Add(1)
 	go callGetBucketsByKeyword("sampleIDs", "samples.id.keyword", &wg)
 
+	// get distribution of assembly IDs
+	wg.Add(1)
+	go callGetBucketsByKeyword("assemblyIDs", "assemblyId.keyword", &wg)
+
 	wg.Wait()
 
 	return c.JSON(http.StatusOK, resultsMap)
@@ -305,7 +312,7 @@ func GetAllVariantIngestionRequests(c echo.Context) error {
 
 func executeGetByIds(c echo.Context, ids []string, isVariantIdQuery bool) error {
 
-	var es, chromosome, lowerBound, upperBound, reference, alternative, genotype = retrieveCommonElements(c)
+	var es, chromosome, lowerBound, upperBound, reference, alternative, genotype, assemblyId = retrieveCommonElements(c)
 
 	// retrieve other query parameters relevent to this 'get' query ---
 	sizeQP := c.QueryParam("size")
@@ -363,7 +370,7 @@ func executeGetByIds(c echo.Context, ids []string, isVariantIdQuery bool) error
 					_id, "", // note : "" is for sampleId
 					reference, alternative,
 					size, sortByPosition,
-					includeSamplesInResultSet, genotype)
+					includeSamplesInResultSet, genotype, assemblyId)
 			} else {
 				// implied sampleId query
 				variantRespDataModel.SampleId = _id
@@ -375,7 +382,7 @@ func executeGetByIds(c echo.Context, ids []string, isVariantIdQuery bool) error
 					"", _id, // note : "" is for variantId
 					reference, alternative,
 					size, sortByPosition,
-					includeSamplesInResultSet, genotype)
+					includeSamplesInResultSet, genotype, assemblyId)
 			}
 
 			// query for each id
@@ -420,7 +427,7 @@ func executeGetByIds(c echo.Context, ids []string, isVariantIdQuery bool) error
 
 func executeCountByIds(c echo.Context, ids []string, isVariantIdQuery bool) error {
 
-	var es, chromosome, lowerBound, upperBound, reference, alternative, genotype = retrieveCommonElements(c)
+	var es, chromosome, lowerBound, upperBound, reference, alternative, genotype, assemblyId = retrieveCommonElements(c)
 
 	respDTO := models.VariantsResponseDTO{}
 	respDTOMux := sync.RWMutex{}
@@ -444,7 +451,7 @@ func executeCountByIds(c echo.Context, ids []string, isVariantIdQuery bool) erro
 				docs = esRepo.CountDocumentsContainerVariantOrSampleIdInPositionRange(es,
 					chromosome, lowerBound, upperBound,
 					_id, "", // note : "" is for sampleId
-					reference, alternative, genotype)
+					reference, alternative, genotype, assemblyId)
 			} else {
 				// implied sampleId query
 				variantRespDataModel.SampleId = _id
@@ -454,7 +461,7 @@ func executeCountByIds(c echo.Context, ids []string, isVariantIdQuery bool) erro
 				docs = esRepo.CountDocumentsContainerVariantOrSampleIdInPositionRange(es,
 					chromosome, lowerBound, upperBound,
 					"", _id, // note : "" is for variantId
-					reference, alternative, genotype)
+					reference, alternative, genotype, assemblyId)
 			}
 
 			variantRespDataModel.Count = int(docs["count"].(float64))
@@ -474,7 +481,7 @@ func executeCountByIds(c echo.Context, ids []string, isVariantIdQuery bool) erro
 	return c.JSON(http.StatusOK, respDTO)
 }
 
-func retrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, int, string, string, constants.GenotypeQuery) {
+func retrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, int, string, string, constants.GenotypeQuery, constants.AssemblyId) {
 	es := c.(*contexts.GohanContext).Es7Client
 
 	chromosome := c.QueryParam("chromosome")
@@ -519,5 +526,11 @@ func retrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int,
 		}
 	}
 
-	return es, chromosome, lowerBound, upperBound, reference, alternative, genotype
+	assemblyId := a.Unknown
+	assemblyIdQP := c.QueryParam("assemblyId")
+	if len(assemblyIdQP) > 0 && a.IsKnownAssemblyId(assemblyIdQP) {
+		assemblyId = a.CastToAssemblyId(assemblyIdQP)
+	}
+
+	return es, chromosome, lowerBound, upperBound, reference, alternative, genotype, assemblyId
 }

src/api/repositories/elasticsearch/main.go

@@ -23,7 +23,8 @@ func GetDocumentsContainerVariantOrSampleIdInPositionRange(es *elasticsearch.Cli
 	variantId string, sampleId string,
 	reference string, alternative string,
 	size int, sortByPosition c.SortDirection,
-	includeSamplesInResultSet bool, genotype c.GenotypeQuery) map[string]interface{} {
+	includeSamplesInResultSet bool,
+	genotype c.GenotypeQuery, assemblyId c.AssemblyId) map[string]interface{} {
 
 	// begin building the request body.
 	mustMap := []map[string]interface{}{{
@@ -60,6 +61,16 @@ func GetDocumentsContainerVariantOrSampleIdInPositionRange(es *elasticsearch.Cli
 		}
 	}
 
+	if assemblyId != "" {
+		mustMap = append(mustMap, map[string]interface{}{
+			"match": map[string]interface{}{
+				"assemblyId": map[string]interface{}{
+					"query": assemblyId,
+				},
+			},
+		})
+	}
+
 	rangeMapSlice := []map[string]interface{}{}
 
 	// TODO: make upperbound and lowerbound nilable, somehow?
@@ -227,7 +238,8 @@ func GetDocumentsContainerVariantOrSampleIdInPositionRange(es *elasticsearch.Cli
 func CountDocumentsContainerVariantOrSampleIdInPositionRange(es *elasticsearch.Client,
 	chromosome string, lowerBound int, upperBound int,
 	variantId string, sampleId string,
-	reference string, alternative string, genotype c.GenotypeQuery) map[string]interface{} {
+	reference string, alternative string,
+	genotype c.GenotypeQuery, assemblyId c.AssemblyId) map[string]interface{} {
 
 	// begin building the request body.
 	mustMap := []map[string]interface{}{{
@@ -264,6 +276,15 @@ func CountDocumentsContainerVariantOrSampleIdInPositionRange(es *elasticsearch.C
 		}
 	}
 
+	if assemblyId != "" {
+		mustMap = append(mustMap, map[string]interface{}{
+			"match": map[string]interface{}{
+				"assemblyId": map[string]interface{}{
+					"query": assemblyId,
+				},
+			},
+		})
+	}
 	rangeMapSlice := []map[string]interface{}{}
 
 	// TODO: make upperbound and lowerbound nilable, somehow?

src/api/services/ingestion.go

@@ -229,7 +229,7 @@ func (i *IngestionService) UploadVcfGzToDrs(gzippedFilePath string, gzipStream *
 	return id
 }
 
-func (i *IngestionService) ProcessVcf(vcfFilePath string, drsFileId string) {
+func (i *IngestionService) ProcessVcf(vcfFilePath string, drsFileId string, assemblyId constants.AssemblyId) {
 	f, err := os.Open(vcfFilePath)
 	if err != nil {
 		fmt.Println("Failed to open file - ", err)
@@ -277,6 +277,7 @@ func (i *IngestionService) ProcessVcf(vcfFilePath string, drsFileId string) {
 			tmpVariantMapMutex := sync.RWMutex{}
 
 			tmpVariant["fileId"] = drsFileId
+			tmpVariant["assemblyId"] = assemblyId
 
 			var rowWg sync.WaitGroup
 			rowWg.Add(len(rowComponents))

src/tests/integration/api/api_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"api/models"
 	c "api/models/constants"
+	a "api/models/constants/assembly-id"
 	gq "api/models/constants/genotype-query"
 	s "api/models/constants/sort"
 	z "api/models/constants/zygosity"
@@ -95,7 +96,6 @@ func TestCanGetVariantsWithoutSamplesInResultset(t *testing.T) {
 	for _, dtoResponse := range allDtoResponses {
 		firstDataPointResults := dtoResponse.Data[0].Results
 		assert.Nil(t, firstDataPointResults[0].Samples)
-
 	}
 }
 
@@ -241,9 +241,9 @@ func runAndValidateGenotypeQueryResults(_t *testing.T, genotypeQuery c.GenotypeQ
 	})
 }
 
-func buildQueryAndMakeGetVariantsCall(chromosome string, sampleId string, includeSamples bool, sortByPosition c.SortDirection, genotype string, _t *testing.T, _cfg *models.Config) models.VariantsResponseDTO {
+func buildQueryAndMakeGetVariantsCall(chromosome string, sampleId string, includeSamples bool, sortByPosition c.SortDirection, genotype string, assemblyId c.AssemblyId, _t *testing.T, _cfg *models.Config) models.VariantsResponseDTO {
 
-	queryString := fmt.Sprintf("?chromosome=%s&ids=%s&includeSamplesInResultSet=%t&sortByPosition=%s&genotype=%s", chromosome, sampleId, includeSamples, sortByPosition, genotype)
+	queryString := fmt.Sprintf("?chromosome=%s&ids=%s&includeSamplesInResultSet=%t&sortByPosition=%s&genotype=%s&assemblyId=%s", chromosome, sampleId, includeSamples, sortByPosition, genotype, assemblyId)
 	url := fmt.Sprintf(VariantsGetBySampleIdsPathWithQueryString, _cfg.Api.Url, queryString)
 
 	return makeGetVariantsCall(url, _t)
@@ -290,12 +290,14 @@ func getVariantsOverview(_t *testing.T, _cfg *models.Config) map[string]interfac
 	return overviewRespJson
 }
 
-func getChromsAndSampleIDs(chromosomeStruct interface{}, sampleIdsStruct interface{}) [][]string {
+func getOverviewResultCombinations(chromosomeStruct interface{}, sampleIdsStruct interface{}, assemblyIdsStruct interface{}) [][]string {
 	var allCombinations = [][]string{}
 
 	for i, _ := range chromosomeStruct.(map[string]interface{}) {
 		for j, _ := range sampleIdsStruct.(map[string]interface{}) {
-			allCombinations = append(allCombinations, []string{i, j})
+			for k, _ := range assemblyIdsStruct.(map[string]interface{}) {
+				allCombinations = append(allCombinations, []string{i, j, k})
+			}
 		}
 	}
 
@@ -309,21 +311,22 @@ func getAllDtosOfVariousCombinationsOfChromosomesAndSampleIds(_t *testing.T, inc
 	overviewJson := getVariantsOverview(_t, cfg)
 	assert.NotNil(_t, overviewJson)
 
-	chromSampleIdCombinations := getChromsAndSampleIDs(overviewJson["chromosomes"], overviewJson["sampleIDs"])
+	overviewCombinations := getOverviewResultCombinations(overviewJson["chromosomes"], overviewJson["sampleIDs"], overviewJson["assemblyIDs"])
 
 	allDtoResponses := []models.VariantsResponseDTO{}
 	allDtoResponsesMux := sync.RWMutex{}
 
 	var combWg sync.WaitGroup
-	for _, combination := range chromSampleIdCombinations {
+	for _, combination := range overviewCombinations {
 		combWg.Add(1)
 		go func(_wg *sync.WaitGroup, _combination []string) {
 			defer _wg.Done()
 
 			chrom := _combination[0]
 			sampleId := _combination[1]
+			assemblyId := a.CastToAssemblyId(_combination[2])
 
-			dto := buildQueryAndMakeGetVariantsCall(chrom, sampleId, includeSamples, sortByPosition, genotype, _t, cfg)
+			dto := buildQueryAndMakeGetVariantsCall(chrom, sampleId, includeSamples, sortByPosition, genotype, assemblyId, _t, cfg)
 			assert.Equal(_t, 1, len(dto.Data))
 
 			allDtoResponsesMux.Lock()