[#230] Excluding search in tests (#259)

* Created functions looking for "tests" in paths * Added tests * Added checkbox for exclude tests * Added total matches in snippet info. * Edited tests * [#230] Excluding search in tests: - fixed tests
aelve · Aug 13, 2019 · d1fccc0 · d1fccc0
1 parent 7bd2aef
commit d1fccc0
Show file tree

Hide file tree

Showing 24 changed files with 177 additions and 36 deletions.
diff --git a/core/src/main/scala/codesearch/core/search/HaskellSearch.scala b/core/src/main/scala/codesearch/core/search/HaskellSearch.scala
@@ -8,4 +8,6 @@ class HaskellSearch(val cindexDir: СindexDirectory) extends Search {
   override protected def extensions: Extensions = HaskellExtensions
   override protected def buildRepUrl(packageName: String, version: String): String =
     s"https://hackage.haskell.org/package/$packageName-$version"
+  def isTestInPath(path: String): Boolean =
+    path.matches(".*/((?i)(test|tests|testsuite|testsuites|test-suite|test-suites))/.*")
 }
diff --git a/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala b/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala
@@ -8,4 +8,5 @@ class JavaScriptSearch(val cindexDir: СindexDirectory) extends Search {
   override protected def extensions: Extensions = JavaScriptExtensions
   override protected def buildRepUrl(packageName: String, version: String): String =
     s"https://www.npmjs.com/package/$packageName/v/$version"
+  def isTestInPath(path: String): Boolean = path.matches(".*((?i)(test|spec|tests)).*")
 }
diff --git a/core/src/main/scala/codesearch/core/search/RubySearch.scala b/core/src/main/scala/codesearch/core/search/RubySearch.scala
@@ -8,4 +8,5 @@ class RubySearch(val cindexDir: СindexDirectory) extends Search {
   override protected def extensions: Extensions = RubyExtensions
   override protected def buildRepUrl(packageName: String, version: String): String =
     s"https://rubygems.org/gems/$packageName/versions/$version"
+  def isTestInPath(path: String): Boolean = path.contains("/spec/") || path.contains("/test/")
 }
diff --git a/core/src/main/scala/codesearch/core/search/RustSearch.scala b/core/src/main/scala/codesearch/core/search/RustSearch.scala
@@ -8,4 +8,5 @@ class RustSearch(val cindexDir: СindexDirectory) extends Search {
   override protected def extensions: Extensions = RustExtensions
   override protected def buildRepUrl(packageName: String, version: String): String =
     s"https://docs.rs/crate/$packageName/$version"
+  def isTestInPath(path: String): Boolean = path.contains("/tests/")
 }
diff --git a/core/src/main/scala/codesearch/core/search/Search.scala b/core/src/main/scala/codesearch/core/search/Search.scala
@@ -17,13 +17,15 @@ import fs2.{Pipe, Stream}
 import io.chrisdavenport.log4cats.SelfAwareStructuredLogger
 import io.chrisdavenport.log4cats.slf4j.Slf4jLogger
 import codesearch.core.regex.RegexConstructor
+import codesearch.core.syntax.stream._
 
 import scala.sys.process.Process
 
 trait Search {
 
   protected def extensions: Extensions
   protected val logger: SelfAwareStructuredLogger[IO] = Slf4jLogger.unsafeCreate[IO]
+  def isTestInPath(path: String): Boolean
 
   def cindexDir: СindexDirectory
 
@@ -32,16 +34,25 @@ trait Search {
   def search(request: SearchRequest): IO[CSearchPage] = {
     for {
       lines <- csearch(request)
-      results <- Stream
+
+      snippetsInfo = Stream
         .emits(lines)
         .through(SnippetsGrouper.groupLines(snippetConfig))
+
+      filteredSnippetsInfo = if (request.withoutTests)
+        snippetsInfo.filterNot(snippetInfo => isTestInPath(snippetInfo.filePath))
+      else
+        snippetsInfo
+
+      results <- filteredSnippetsInfo
         .drop(snippetConfig.pageSize * (request.page - 1))
         .take(snippetConfig.pageSize)
         .evalMap(createSnippet)
         .through(groupByPackage)
         .compile
         .toList
-    } yield CSearchPage(results.sortBy(_.pack.name), lines.size)
+      totalMatches = filteredSnippetsInfo.fold(0)((total, snippet) => total + snippet.totalMatches).compile.toList.last
+    } yield CSearchPage(results.sortBy(_.pack.name), totalMatches)
   }
 
   /**

diff --git a/core/src/main/scala/codesearch/core/search/SearchRequest.scala b/core/src/main/scala/codesearch/core/search/SearchRequest.scala
@@ -10,6 +10,7 @@ import com.softwaremill.sttp._
   * @param spaceInsensitive space insensitive search flag
   * @param preciseMatch precise match flag
   * @param sourcesOnly sources only flag
+  * @param withoutTests search without tests
   * @param page next pagination
   */
 case class SearchRequest(
@@ -21,6 +22,7 @@ case class SearchRequest(
     spaceInsensitive: Boolean,
     preciseMatch: Boolean,
     sourcesOnly: Boolean,
+    withoutTests: Boolean,
     page: Int
 ) {
 
@@ -32,7 +34,7 @@ case class SearchRequest(
     def stringify(x: Boolean): String = if (x) "on" else "off"
 
     uri"$host/$lang/search?query=$query&filter=$filter&filePath=$filePath&insensitive=${stringify(insensitive)}&space=${stringify(
-      spaceInsensitive)}&precise=${stringify(preciseMatch)}&sources=${stringify(sourcesOnly)}"
+      spaceInsensitive)}&precise=${stringify(preciseMatch)}&sources=${stringify(sourcesOnly)}&withoutTests=${stringify(withoutTests)}"
   }
 }
 
@@ -46,6 +48,7 @@ object SearchRequest {
       spaceInsensitive: String,
       preciseMatch: String,
       sourcesOnly: String,
+      withoutTests: String,
       page: String
   ): SearchRequest = {
     SearchRequest(
@@ -57,6 +60,7 @@ object SearchRequest {
       isEnabled(spaceInsensitive),
       isEnabled(preciseMatch),
       isEnabled(sourcesOnly),
+      isEnabled(withoutTests),
       page.toInt,
     )
   }

diff --git a/core/src/main/scala/codesearch/core/search/SnippetsGrouper.scala b/core/src/main/scala/codesearch/core/search/SnippetsGrouper.scala
@@ -13,7 +13,7 @@ object SnippetsGrouper {
     * @param filePath absolute path to file
     * @param lines numbers of matched lines in file
     */
-  case class SnippetInfo(filePath: String, lines: NonEmptyVector[Int])
+  case class SnippetInfo(filePath: String, lines: NonEmptyVector[Int], totalMatches: Int)
 
   private case class ResultRow(path: String, lineNumber: Int)
 
@@ -40,12 +40,13 @@ object SnippetsGrouper {
       snippets.lastOption match {
         case Some(snippet) =>
           if (row.lineNumber < snippet.lines.last + config.linesAfter) {
-            snippets.init :+ snippet.copy(lines = snippet.lines :+ row.lineNumber)
+            snippets.init :+ snippet.copy(lines = snippet.lines :+ row.lineNumber,
+                                          totalMatches = snippet.totalMatches + 1)
           } else {
-            snippets :+ SnippetInfo(row.path, NonEmptyVector.one(row.lineNumber))
+            snippets :+ SnippetInfo(row.path, NonEmptyVector.one(row.lineNumber), 1)
           }
         case None =>
-          snippets :+ SnippetInfo(row.path, NonEmptyVector.one(row.lineNumber))
+          snippets :+ SnippetInfo(row.path, NonEmptyVector.one(row.lineNumber), 1)
       }
     }
   }

diff --git a/core/src/main/scala/codesearch/core/syntax/stream.scala b/core/src/main/scala/codesearch/core/syntax/stream.scala
@@ -5,11 +5,13 @@ import fs2.Stream
 import cats.syntax.functor._
 
 object stream {
-  implicit class StreamOps[F[_]: Functor, A](val stream: Stream[F, A]) {
-    def filterM(f: A => F[Boolean]): Stream[F, A] =
+  implicit class StreamOps[F[_], A](val stream: Stream[F, A]) {
+    def filterM(f: A => F[Boolean])(implicit F: Functor[F]): Stream[F, A] =
       stream.evalMap(x => f(x).map(x -> _)).collect { case (value, true) => value }
 
-    def filterNotM(f: A => F[Boolean]): Stream[F, A] =
+    def filterNotM(f: A => F[Boolean])(implicit F: Functor[F]): Stream[F, A] =
       stream.evalMap(x => f(x).map(x -> _)).collect { case (value, false) => value }
+
+    def filterNot(f: A => Boolean): Stream[F, A] = stream.filter(!f(_))
   }
 }
diff --git a/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala b/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala
@@ -0,0 +1,10 @@
+package codesearch.core.search
+
+class HaskellSearchSpec extends SearchSpecBase {
+  "Checking on the way to tests" - {
+    haskellIsTestInWay(path = "path/to/package/test/", result = true)
+    haskellIsTestInWay(path = "path/to/package/testSUites/", result = true)
+    haskellIsTestInWay(path = "path/to/package/tEsts/", result = true)
+    haskellIsTestInWay(path = "path/to/package/tes-t/", result = false)
+  }
+}
diff --git a/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala b/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala
@@ -0,0 +1,10 @@
+package codesearch.core.search
+
+class JavaScriptSearchSpec extends SearchSpecBase {
+  "Checking on the way to tests" - {
+    javaScriptIsTestInWay(path = "path/to/package/tests/", result = true)
+    javaScriptIsTestInWay(path = "path/to/package/sPEc-class/", result = true)
+    javaScriptIsTestInWay(path = "path/to/package/tEsts.js", result = true)
+    javaScriptIsTestInWay(path = "path/to/package/tes-t/", result = false)
+  }
+}
diff --git a/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala b/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala
@@ -0,0 +1,10 @@
+package codesearch.core.search
+
+class RubySearchSpec extends SearchSpecBase {
+  "Checking on the way to tests" - {
+    rubyIsTestInWay(path = "path/to/package/test/", result = true)
+    rubyIsTestInWay(path = "path/to/package/spec/", result = true)
+    rubyIsTestInWay(path = "path/to/package/tEsts/", result = false)
+    rubyIsTestInWay(path = "path/to/package/tes-t/", result = false)
+  }
+}
diff --git a/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala b/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala
@@ -0,0 +1,10 @@
+package codesearch.core.search
+
+class RustSearchSpec extends SearchSpecBase {
+  "Checking on the way to tests" - {
+    rustIsTestInWay(path = "path/to/package/tests/", result = true)
+    rustIsTestInWay(path = "path/to/package/spec/", result = false)
+    rustIsTestInWay(path = "path/to/package/tEsts/", result = false)
+    rustIsTestInWay(path = "path/to/package/tes-t/", result = false)
+  }
+}
diff --git a/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala b/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala
@@ -0,0 +1,42 @@
+package codesearch.core.search
+
+import java.nio.file.Paths
+
+import codesearch.core.index.directory.{HaskellCindex, JavaScriptCindex, RubyCindex, RustCindex}
+import org.scalatest.{FreeSpec, Matchers}
+
+
+trait SearchSpecBase extends FreeSpec with Matchers{
+  val haskellCindex = HaskellCindex(Paths.get("./index/test/cindex/"))
+  val haskellSearch = new HaskellSearch(haskellCindex)
+  val rubyCindex = RubyCindex(Paths.get("./index/test/cindex/"))
+  val rubySearch = new RubySearch(rubyCindex)
+  val rustCindex = RustCindex(Paths.get("./index/test/cindex/"))
+  val rustSearch = new RustSearch(rustCindex)
+  val javaScriptCindex = JavaScriptCindex(Paths.get("./index/test/cindex/"))
+  val javaScriptSearch = new JavaScriptSearch(javaScriptCindex)
+
+  def haskellIsTestInWay(path: String, result: Boolean): Unit = {
+    path in {
+      haskellSearch.isTestInPath(path) shouldBe result
+    }
+  }
+
+  def rubyIsTestInWay(path: String, result: Boolean): Unit = {
+    path in {
+      rubySearch.isTestInPath(path) shouldBe result
+    }
+  }
+
+  def rustIsTestInWay(path: String, result: Boolean): Unit = {
+    path in {
+      rustSearch.isTestInPath(path) shouldBe result
+    }
+  }
+
+  def javaScriptIsTestInWay(path: String, result: Boolean): Unit = {
+    path in {
+      javaScriptSearch.isTestInPath(path) shouldBe result
+    }
+  }
+}
diff --git a/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala b/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala
@@ -28,7 +28,8 @@ class SnippetsGrouperSpec extends WordSpec with Matchers {
     snippets should contain theSameElementsAs List(
       SnippetInfo(
         filePath = "3models/0.3.0/Graphics/Model/DirectX.hs",
-        lines = NonEmptyVector.of(13, 14, 15, 16, 17)
+        lines = NonEmptyVector.of(13, 14, 15, 16, 17),
+        totalMatches = 5
       )
     )
   }
@@ -51,11 +52,13 @@ class SnippetsGrouperSpec extends WordSpec with Matchers {
     snippets should contain theSameElementsAs List(
       SnippetInfo(
         filePath = "3models/0.3.0/Graphics/Model/DirectX.hs",
-        lines = NonEmptyVector.of(28)
+        lines = NonEmptyVector.of(28),
+        totalMatches = 1
       ),
       SnippetInfo(
         filePath = "3models/0.3.0/Graphics/Model/DirectX.hs",
-        lines = NonEmptyVector.of(39)
+        lines = NonEmptyVector.of(39),
+        totalMatches = 1
       )
     )
   }
@@ -78,11 +81,13 @@ class SnippetsGrouperSpec extends WordSpec with Matchers {
     snippets should contain theSameElementsAs List(
       SnippetInfo(
         filePath = "3models/0.3.0/Graphics/Model/DirectX.hs",
-        lines = NonEmptyVector.of(14)
+        lines = NonEmptyVector.of(14),
+        totalMatches = 1
       ),
       SnippetInfo(
         filePath = "3models/0.3.0/Graphics/Model/Obj.hs",
-        lines = NonEmptyVector.of(16)
+        lines = NonEmptyVector.of(16),
+        totalMatches = 1
       )
     )
   }

diff --git a/core/src/test/scala/integration/IntegrationHaskellSpec.scala b/core/src/test/scala/integration/IntegrationHaskellSpec.scala
@@ -11,9 +11,9 @@ import codesearch.core.meta._
 import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult}
 import codesearch.core.search.{HaskellSearch, Search, SearchRequest}
 import codesearch.core.util.Unarchiver
+import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer}
 import integration.fakes.FakeDownloader
 import org.scalatest.FreeSpec
-import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer}
 
 class IntegrationHaskellSpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase {
 
@@ -48,7 +48,8 @@ class IntegrationHaskellSpec extends FreeSpec with ForAllTestContainer with Inte
         spaceInsensitive = false,
         preciseMatch = false,
         sourcesOnly = false,
-        page = 1
+        page = 1,
+        withoutTests = false
       ),
       1,
       Seq(
@@ -87,7 +88,8 @@ class IntegrationHaskellSpec extends FreeSpec with ForAllTestContainer with Inte
         spaceInsensitive = false,
         preciseMatch = false,
         sourcesOnly = true,
-        page = 1
+        page = 1,
+        withoutTests = false
       ),
       1,
       Seq(

diff --git a/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala b/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala
@@ -11,9 +11,9 @@ import codesearch.core.meta._
 import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult}
 import codesearch.core.search.{JavaScriptSearch, Search, SearchRequest}
 import codesearch.core.util.Unarchiver
+import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer}
 import integration.fakes.FakeDownloader
 import org.scalatest.FreeSpec
-import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer}
 
 class IntegrationJavaScriptSpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase {
 
@@ -48,7 +48,8 @@ class IntegrationJavaScriptSpec extends FreeSpec with ForAllTestContainer with I
         spaceInsensitive = true,
         preciseMatch = true,
         sourcesOnly = true,
-        page = 1
+        page = 1,
+        withoutTests = false
       ),
       2,
       Seq(
@@ -89,7 +90,8 @@ class IntegrationJavaScriptSpec extends FreeSpec with ForAllTestContainer with I
         spaceInsensitive = false,
         preciseMatch = true,
         sourcesOnly = true,
-        page = 1
+        page = 1,
+        withoutTests = false
       ),
       1,
       Seq(

diff --git a/core/src/test/scala/integration/IntegrationRubySpec.scala b/core/src/test/scala/integration/IntegrationRubySpec.scala
@@ -11,9 +11,9 @@ import codesearch.core.meta._
 import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult}
 import codesearch.core.search.{RubySearch, Search, SearchRequest}
 import codesearch.core.util.Unarchiver
+import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer}
 import integration.fakes.FakeDownloader
 import org.scalatest.FreeSpec
-import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer}
 
 class IntegrationRubySpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase {
 
@@ -48,7 +48,8 @@ class IntegrationRubySpec extends FreeSpec with ForAllTestContainer with Integra
         spaceInsensitive = false,
         preciseMatch = false,
         sourcesOnly = false,
-        page = 1
+        page = 1,
+        withoutTests = false
       ),
       1,
       Seq(
@@ -85,7 +86,8 @@ class IntegrationRubySpec extends FreeSpec with ForAllTestContainer with Integra
         spaceInsensitive = false,
         preciseMatch = false,
         sourcesOnly = false,
-        page = 1
+        page = 1,
+        withoutTests = false
       ),
       2,
       Seq(