Skip to content

Commit

Permalink
Merge pull request #526 from digital-preservation/DR2-2045_fixOptionR…
Browse files Browse the repository at this point in the history
…elatedErrorWhenEvaluatingRules

Dr2 2045 fix Option-related error when evaluating rules
  • Loading branch information
techncl authored Jan 21, 2025
2 parents 2b7782d + 6357c5c commit 3cc646d
Show file tree
Hide file tree
Showing 9 changed files with 262 additions and 27 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ abstract class Rule(name: String, val argProviders: ArgProvider*) extends Positi
else fail(columnIndex, row, schema)
}

def argProviderHelper(provider: ArgProvider, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, cellValue: String): (Option[FilePathBase], FilePathBase) = {
val ruleValue = provider.referenceValue(columnIndex, row, schema)

if (columnDefinition.directives.contains(IgnoreCase())) (ruleValue.map(_.toLowerCase), cellValue.toLowerCase) else (ruleValue, cellValue)
}

def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int,
row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,47 +128,36 @@ case class FileExistsRule(pathSubstitutions: List[(String,String)], enforceCaseS

case class InRule(inValue: ArgProvider) extends Rule("in", Seq(inValue): _*) {
override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean = {
val ruleValue = inValue.referenceValue(columnIndex, row, schema)

val (rv, cv) = if (columnDefinition.directives.contains(IgnoreCase())) (ruleValue.get.toLowerCase, cellValue.toLowerCase) else (ruleValue.get, cellValue)
rv contains cv
val (potentialRv, cv) = argProviderHelper(inValue, columnDefinition, columnIndex, row, schema, cellValue)
potentialRv.exists(_.contains(cv))
}
}

case class IsRule(isValue: ArgProvider) extends Rule("is", Seq(isValue): _*) {
override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean = {
val ruleValue = isValue.referenceValue(columnIndex, row, schema)

val (rv, cv) = if (columnDefinition.directives.contains(IgnoreCase())) (ruleValue.get.toLowerCase, cellValue.toLowerCase) else (ruleValue.get, cellValue)
cv == rv
val (potentialRv, cv) = argProviderHelper(isValue, columnDefinition, columnIndex, row, schema, cellValue)
potentialRv.contains(cv)
}
}


case class NotRule(notValue: ArgProvider) extends Rule("not", Seq(notValue): _*) {
override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema,mayBeLast: Option[Boolean] = None): Boolean = {
val ruleValue = notValue.referenceValue(columnIndex, row, schema)

val (rv, cv) = if (columnDefinition.directives.contains(IgnoreCase())) (ruleValue.get.toLowerCase, cellValue.toLowerCase) else (ruleValue.get, cellValue)
cv != rv
val (potentialRv, cv) = argProviderHelper(notValue, columnDefinition, columnIndex, row, schema, cellValue)
!potentialRv.contains(cv)
}
}

case class StartsRule(startsValue: ArgProvider) extends Rule("starts", Seq(startsValue): _*) {
override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean = {
val ruleValue = startsValue.referenceValue(columnIndex, row, schema)

val (rv, cv) = if (columnDefinition.directives.contains(IgnoreCase())) (ruleValue.get.toLowerCase, cellValue.toLowerCase) else (ruleValue.get, cellValue)
cv startsWith rv
val (potentialRv, cv) = argProviderHelper(startsValue, columnDefinition, columnIndex, row, schema, cellValue)
potentialRv.exists(rv => cv.startsWith(rv))
}
}

case class EndsRule(endsValue: ArgProvider) extends Rule("ends", Seq(endsValue): _*) {
override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean = {
val ruleValue = endsValue.referenceValue(columnIndex, row, schema)

val (rv, cv) = if (columnDefinition.directives.contains(IgnoreCase())) (ruleValue.get.toLowerCase, cellValue.toLowerCase) else (ruleValue.get, cellValue)
cv endsWith rv
val (potentialRv, cv) = argProviderHelper(endsValue, columnDefinition, columnIndex, row, schema, cellValue)
potentialRv.exists(rv => cv.endsWith(rv))
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright (c) 2013, The National Archives <[email protected]>
* https://www.nationalarchives.gov.uk
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package uk.gov.nationalarchives.csv.validator.schema.v1_0

import cats.data.Validated
import org.junit.runner.RunWith
import org.specs2.mutable.Specification
import org.specs2.runner.JUnitRunner
import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row}
import uk.gov.nationalarchives.csv.validator.schema._

@RunWith(classOf[JUnitRunner])
class EndsRuleSpec extends Specification {

"EndsRule with a string literal behaviour" should {
val globalDirsOne = List(TotalColumns(1))

"succeed if cell ends with endsRule value" in {
val endsRule = EndsRule(Literal(Some("world")))

endsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if cell does not end with endsRule value" in {
val endsRule = EndsRule(Literal(Some("hello world")))
endsRule.evaluate(0, Row(List(Cell("hello")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """ends("hello world") fails for line: 1, column: column1, value: "hello""""
}
}

"succeed if endsRule is the same as value" in {
val endsRule = EndsRule(Literal(Some("hello world")))
endsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"succeed if endsRule's column reference does exist" in {
val endsRule = EndsRule(ColumnReference(NamedColumnIdentifier("column1")))

endsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if endsRule's column reference doesn't exist" in {
val endsRule = EndsRule(ColumnReference(NamedColumnIdentifier("nonExistentColumn")))

endsRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """ends($nonExistentColumn) fails for line: 1, column: column1, value: "hello world today""""
}
}

"succeed with @ignoreCase" in {
val endsRule = EndsRule(Literal(Some("hello world")))
endsRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) mustEqual Validated.Valid(true)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,34 @@ class InRuleSpec extends Specification {
val globalDirsOne = List(TotalColumns(1))

"succeed if inRule is embedded in value" in {
val inRule = InRule(Literal(Some("myhello world today")))
val inRule = InRule(Literal(Some("hello world today")))
inRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if inRule is not in value" in {
val inRule = InRule(Literal(Some("hello world")))

inRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """in("hello world") fails for line: 1, column: column1, value: "hello world today""""
}
}

"succeed if inRule is the same as value" in {
val inRule = InRule(Literal(Some("hello world")))
inRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if inRule is not in value" in {
val inRule = InRule(Literal(Some("hello world")))
"succeed if inRule's column reference does exist" in {
val inRule = InRule(ColumnReference(NamedColumnIdentifier("column1")))

inRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if inRule's column reference doesn't exist" in {
val inRule = InRule(ColumnReference(NamedColumnIdentifier("nonExistentColumn")))

inRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """in("hello world") fails for line: 1, column: column1, value: "hello world today""""
case Validated.Invalid(messages) => messages.head mustEqual """in($nonExistentColumn) fails for line: 1, column: column1, value: "hello world today""""
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Copyright (c) 2013, The National Archives <[email protected]>
* https://www.nationalarchives.gov.uk
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package uk.gov.nationalarchives.csv.validator.schema.v1_0

import cats.data.Validated
import org.junit.runner.RunWith
import org.specs2.mutable.Specification
import org.specs2.runner.JUnitRunner
import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row}
import uk.gov.nationalarchives.csv.validator.schema._

@RunWith(classOf[JUnitRunner])
class IsRuleSpec extends Specification {

"IsRule with a string literal behaviour" should {
val globalDirsOne = List(TotalColumns(1))

"succeed if isRule is the same as value" in {
val isRule = IsRule(Literal(Some("hello world")))
isRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if isRule is not the same as value" in {
val isRule = IsRule(Literal(Some("completely different value")))
isRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """is("completely different value") fails for line: 1, column: column1, value: "hello world""""
}
}

"fail if isRule is embedded in value" in {
val isRule = IsRule(Literal(Some("hello world today")))
isRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """is("hello world today") fails for line: 1, column: column1, value: "hello world""""
}
}

"fail if isRule is not in value" in {
val isRule = IsRule(Literal(Some("hello world")))
isRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """is("hello world") fails for line: 1, column: column1, value: "hello world today""""
}
}

"succeed with @ignoreCase" in {
val isRule = IsRule(Literal(Some("hello world")))
isRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) mustEqual Validated.Valid(true)
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Copyright (c) 2013, The National Archives <[email protected]>
* https://www.nationalarchives.gov.uk
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package uk.gov.nationalarchives.csv.validator.schema.v1_0

import cats.data.Validated
import org.junit.runner.RunWith
import org.specs2.mutable.Specification
import org.specs2.runner.JUnitRunner
import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row}
import uk.gov.nationalarchives.csv.validator.schema._

@RunWith(classOf[JUnitRunner])
class NotRuleSpec extends Specification {

"NotRule with a string literal behaviour" should {
val globalDirsOne = List(TotalColumns(1))

"succeed if notRule is not the same as value" in {
val notRule = NotRule(Literal(Some("completely different value")))
notRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"succeed if notRule is the similar to value" in {
val notRule = NotRule(Literal(Some("hello world ")))
notRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if notRule is the same as value" in {
val notRule = NotRule(Literal(Some("hello world")))
notRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """not("hello world") fails for line: 1, column: column1, value: "hello world""""
}
}

"fail with @ignoreCase" in {
val notRule = NotRule(Literal(Some("hello world")))
notRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """not("hello world") fails for line: 1, column: column1, value: "hello WORLD""""
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,9 @@ class OrRuleSpec extends Specification {

val orRule = OrRule(leftInRule, rightInRule)

orRule.evaluate(0, Row(List(Cell("UK")), 1), schema) must throwA[NoSuchElementException]
orRule.evaluate(0, Row(List(Cell("UK")), 1), schema) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """in($ConfigurableCountry) or in("France") fails for line: 1, column: Country, value: "UK""""
}
}

"succeed when 3 'or' rules valid for right rule" in {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ class SchemaParserSpecs extends SchemaSpecBase {
LastName: @IgnoreCase regex ("[a]")"""

parse(new StringReader(schema)) must beLike {
case Failure(messages, _) => messages mustEqual "Invalid column directive"
case Failure(messages, _) => messages mustEqual "Invalid column definition"
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright (c) 2013, The National Archives <[email protected]>
* https://www.nationalarchives.gov.uk
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package uk.gov.nationalarchives.csv.validator.schema.v1_0

import cats.data.Validated
import org.junit.runner.RunWith
import org.specs2.mutable.Specification
import org.specs2.runner.JUnitRunner
import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row}
import uk.gov.nationalarchives.csv.validator.schema._

@RunWith(classOf[JUnitRunner])
class StartsRuleSpec extends Specification {

"StartsRule with a string literal behaviour" should {
val globalDirsOne = List(TotalColumns(1))

"succeed if cell starts with startsRule value" in {
val startsRule = StartsRule(Literal(Some("hello world")))

startsRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if cell does not start with startsRule value" in {
val startsRule = StartsRule(Literal(Some("hello world today")))
startsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """starts("hello world today") fails for line: 1, column: column1, value: "hello world""""
}
}

"succeed if startsRule is the same as value" in {
val startsRule = StartsRule(Literal(Some("hello world")))
startsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"succeed if startsRule's column reference does exist" in {
val startsRule = StartsRule(ColumnReference(NamedColumnIdentifier("column1")))

startsRule.evaluate(0, Row(List(Cell("hello world")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true)
}

"fail if startsRule's column reference doesn't exist" in {
val startsRule = StartsRule(ColumnReference(NamedColumnIdentifier("nonExistentColumn")))

startsRule.evaluate(0, Row(List(Cell("hello world today")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike {
case Validated.Invalid(messages) => messages.head mustEqual """starts($nonExistentColumn) fails for line: 1, column: column1, value: "hello world today""""
}
}

"succeed with @ignoreCase" in {
val startsRule = StartsRule(Literal(Some("hello world")))
startsRule.evaluate(0, Row(List(Cell("hello WORLD")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"), Nil, List(IgnoreCase()))))) mustEqual Validated.Valid(true)
}
}
}

0 comments on commit 3cc646d

Please sign in to comment.