Merge pull request #1204 from ProjectSidewalk/develop

v4.3.8

app/controllers/AdminController.scala

@@ -183,19 +183,19 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
     */
   def getRegionNegativeLabelCounts() = UserAwareAction.async { implicit request =>
 
-      val neighborhoods = RegionCompletionTable.selectAllNamedNeighborhoodCompletions
-
-      val features: List[JsObject] = neighborhoods.map {neighborhood =>
-       val labelResults = LabelTable.selectNegativeLabelCountsByRegionId(neighborhood.regionId)
-       Json.obj(
-            "region_id" -> neighborhood.regionId,
-            "labels" -> Json.toJson(labelResults.toMap)
-       )
-      }
+    val neighborhoods = RegionCompletionTable.selectAllNamedNeighborhoodCompletions
+
+    val features: List[JsObject] = neighborhoods.map {neighborhood =>
+      val labelResults = LabelTable.selectNegativeLabelCountsByRegionId(neighborhood.regionId)
+      Json.obj(
+        "region_id" -> neighborhood.regionId,
+        "labels" -> Json.toJson(labelResults.toMap)
+      )
+    }
 
-      val jsonObjectList = features.map(x => Json.toJson(x))
+    val jsonObjectList = features.map(x => Json.toJson(x))
 
-      Future.successful(Ok(JsArray(jsonObjectList)))
+    Future.successful(Ok(JsArray(jsonObjectList)))
 
   }
 
@@ -266,34 +266,49 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
   }
 
   /**
-  * Get all auditing times
-  *
-  * @return
-  */
-def getAuditTimes() = UserAwareAction.async { implicit request =>
-  if (isAdmin(request.identity)) {
-        val auditTimes = AuditTaskInteractionTable.selectAllAuditTimes().map(auditTime =>
-          Json.obj("user_id" -> auditTime.userId, "time" -> auditTime.duration, "ip_address" -> auditTime.ipAddress))
+    * Get all auditing times
+    *
+    * @return
+    */
+  def getAuditTimes() = UserAwareAction.async { implicit request =>
+    if (isAdmin(request.identity)) {
+      val auditTimes = AuditTaskInteractionTable.selectAllAuditTimes().map(auditTime =>
+        Json.obj("user_id" -> auditTime.userId, "time" -> auditTime.duration, "ip_address" -> auditTime.ipAddress))
       Future.successful(Ok(JsArray(auditTimes)))
-  } else {
-    Future.successful(Redirect("/"))
+    } else {
+      Future.successful(Redirect("/"))
+    }
   }
-}
 
-/**
-  * Get all anonymous auditing times
-  *
-  * @return
-  */
-def getAnonAuditTimes() = UserAwareAction.async { implicit request =>
-  if (isAdmin(request.identity)) {
+  /**
+    * Get all auditing times for Turkers
+    *
+    * @return
+    */
+  def getTurkerAuditTimes() = UserAwareAction.async { implicit request =>
+    if (isAdmin(request.identity)) {
+      val auditTimes = AuditTaskInteractionTable.selectAllTurkerAuditTimes().map(auditTime =>
+        Json.obj("user_id" -> auditTime.userId, "time" -> auditTime.duration, "ip_address" -> auditTime.ipAddress))
+      Future.successful(Ok(JsArray(auditTimes)))
+    } else {
+      Future.successful(Redirect("/"))
+    }
+  }
+
+  /**
+    * Get all anonymous auditing times
+    *
+    * @return
+    */
+  def getAnonAuditTimes() = UserAwareAction.async { implicit request =>
+    if (isAdmin(request.identity)) {
       val anonAuditTimes = AuditTaskInteractionTable.selectAllAnonAuditTimes().map(auditTime =>
         Json.obj("user_id" -> auditTime.userId, "time" -> auditTime.duration, "ip_address" -> auditTime.ipAddress))
       Future.successful(Ok(JsArray(anonAuditTimes)))
-  } else {
-    Future.successful(Redirect("/"))
+    } else {
+      Future.successful(Redirect("/"))
+    }
   }
-}
 
 
   /**
@@ -424,6 +439,14 @@ def getAnonAuditTimes() = UserAwareAction.async { implicit request =>
     Future.successful(Ok(json))
   }
 
+  def getAllTurkerUserLabelCounts = UserAwareAction.async { implicit request =>
+    val labelCounts = LabelTable.getLabelCountsPerTurkerUser
+    val json = Json.arr(labelCounts.map(x => Json.obj(
+      "user_id" -> x._1, "count" -> x._2, "is_researcher" -> UserRoleTable.isResearcher(UUID.fromString(x._1))
+    )))
+    Future.successful(Ok(json))
+  }
+
   def getAllAnonUserLabelCounts = UserAwareAction.async { implicit request =>
     val labelCounts = LabelTable.getLabelCountsPerAnonUser
     val json = Json.arr(labelCounts.map(x => Json.obj(
@@ -526,7 +549,7 @@ def getAnonAuditTimes() = UserAwareAction.async { implicit request =>
               Future.successful(BadRequest("No user has this user ID"))
           }
         } else {
-          Future.successful(Redirect("/"))   
+          Future.successful(Redirect("/"))
         }
       }
     )

app/controllers/UserProfileController.scala

@@ -266,6 +266,14 @@ class UserProfileController @Inject() (implicit val env: Environment[User, Sessi
     Future.successful(Ok(json))
   }
 
+  def getTurkerCompletedMissionCounts = UserAwareAction.async { implicit request =>
+    val missionCounts = MissionTable.selectMissionCountsPerTurkerUser
+    val json = Json.arr(missionCounts.map(x =>
+      Json.obj("user_id" -> x._1, "count" -> x._2, "is_researcher" -> UserRoleTable.isResearcher(UUID.fromString(x._1)))
+    ))
+    Future.successful(Ok(json))
+  }
+
   def isTurker = UserAwareAction.async { implicit request =>
     request.identity match {
       case Some(user) =>

app/models/audit/AuditTaskInteractionTable.scala

@@ -154,6 +154,36 @@ def selectAllAuditTimes(): List[UserAuditTime] = db.withSession { implicit sessi
     auditTimes
 }
 
+  /**
+    * Select all audit task interaction times for Turker users
+    * @return
+    */
+  def selectAllTurkerAuditTimes(): List[UserAuditTime] = db.withSession { implicit session =>
+    val selectAuditTimesQuery = Q.query[String, UserAuditTime](
+      """SELECT user_audit_times.user_id,
+        |  CAST(extract( second from SUM(diff) ) /60 +
+        |       extract( minute from SUM(diff) ) +
+        |       extract( hour from SUM(diff) ) * 60 AS decimal(10,2)) AS total_time_spent_auditing,
+        |  NULL
+        |FROM (
+        |       SELECT audit_task.user_id, (timestamp - LAG(timestamp, 1) OVER(PARTITION BY audit_task.user_id ORDER BY timestamp)) AS diff
+        |       FROM audit_task_interaction
+        |         LEFT JOIN audit_task
+        |           ON audit_task.audit_task_id = audit_task_interaction.audit_task_id
+        |         INNER JOIN user_role
+        |           ON audit_task.user_id = user_role.user_id
+        |         INNER JOIN sidewalk.role
+        |           ON user_role.role_id = sidewalk.role.role_id
+        |       WHERE action = 'ViewControl_MouseDown'
+        |             AND sidewalk.role.role = ?
+        |     ) user_audit_times
+        |WHERE diff < '00:05:00.000' AND diff > '00:00:00.000'
+        |GROUP BY user_id;""".stripMargin
+    )
+    val auditTimes: List[UserAuditTime] = selectAuditTimesQuery("Turker").list
+    auditTimes
+  }
+
 /**
   * Select all audit task interaction times for anonymous users
   *

app/models/label/LabelTable.scala

@@ -5,7 +5,9 @@ import java.util.UUID
 
 import com.vividsolutions.jts.geom.LineString
 import models.audit.{AuditTask, AuditTaskEnvironmentTable, AuditTaskInteraction, AuditTaskTable}
+import models.daos.slick.DBTableDefinitions.UserTable
 import models.region.RegionTable
+import models.user.{RoleTable, UserRoleTable}
 import models.utils.MyPostgresDriver.simple._
 import play.api.Play.current
 import play.api.libs.json.{JsObject, Json}
@@ -81,6 +83,8 @@ object LabelTable {
   val labelPoints = TableQuery[LabelPointTable]
   val regions = TableQuery[RegionTable]
   val severities = TableQuery[ProblemSeverityTable]
+  val userRoles = TableQuery[UserRoleTable]
+  val roleTable = TableQuery[RoleTable]
 
   val labelsWithoutDeleted = labels.filter(_.deleted === false)
   val neighborhoods = regions.filter(_.deleted === false).filter(_.regionTypeId === 2)
@@ -94,7 +98,6 @@ object LabelTable {
 
   val anonIps = anonUsersAudits.groupBy(_._1).map{case(ip,group)=>ip}
 
-
   case class LabelCountPerDay(date: String, count: Int)
 
   case class LabelMetadata(labelId: Int, gsvPanoramaId: String, heading: Float, pitch: Float, zoom: Int,
@@ -652,4 +655,21 @@ object LabelTable {
     // missions ended up in the completedMissions query.
     labelCounts.map{pair => (pair._1.get, pair._2.getOrElse(0))}
   }
+
+  /**
+    * Select label counts per turker user
+    */
+  def getLabelCountsPerTurkerUser: List[(String, Int)] = db.withSession { implicit session =>
+    val turkerUsers = UserRoleTable.getUsersByType("Turker")
+
+    val turkerAudits = for {
+      _audits <- completedAudits
+      _turkerAudits <- turkerUsers if _audits.userId === _turkerAudits.userId
+      _labels <- labelsWithoutDeleted if _audits.auditTaskId === _labels.auditTaskId
+    } yield _turkerAudits.userId
+
+
+    // counts the number of tasks for each user
+    turkerAudits.groupBy(l => l).map{ case (uid, group) => (uid, group.length)}.list
+  }
 }

app/models/mission/MissionTable.scala

@@ -3,8 +3,10 @@ package models.mission
 import java.util.UUID
 
 import models.daos.slick.DBTableDefinitions.UserTable
+import models.label.LabelTable.{roleTable, userRoles}
 import models.utils.MyPostgresDriver.simple._
 import models.region._
+import models.user.UserRoleTable
 import play.api.Play.current
 import play.api.libs.json.{JsObject, Json}
 
@@ -53,6 +55,8 @@ object MissionTable {
   val db = play.api.db.slick.DB
   val missions = TableQuery[MissionTable]
   val missionUsers = TableQuery[MissionUserTable]
+  val turkerUsers = UserRoleTable.getUsersByType("Turker")
+
   val users = TableQuery[UserTable]
   val regionProperties = TableQuery[RegionPropertyTable]
   val regions = TableQuery[RegionTable]
@@ -235,4 +239,21 @@ object MissionTable {
 
     _missions.groupBy(m => m).map{ case(id, group) => (id, group.length)}.list
   }
+
+  /**
+    * Select mission counts for turkers
+    *
+    * @ List[(user_id,count)]
+    */
+  def selectMissionCountsPerTurkerUser: List[(String, Int)] = db.withSession { implicit session =>
+    val missionTurkers = for {
+      (_missionusers, _turkerusers) <- missionUsers.innerJoin(turkerUsers).on(_.userId === _.userId)
+    } yield _missionusers
+
+    val _missions = for {
+      (_missions, _missionUsers) <- missionsWithoutDeleted.innerJoin(missionTurkers).on(_.missionId === _.missionId)
+    } yield _missionUsers.userId
+
+    _missions.groupBy(m => m).map{ case(id, group) => (id, group.length)}.list
+  }
 }

app/models/user/UserRoleTable.scala

@@ -1,5 +1,6 @@
 package models.user
 
+import models.daos.slick.DBTableDefinitions.{DBUser, UserTable}
 import models.utils.MyPostgresDriver.simple._
 import play.api.Play.current
 import java.util.UUID
@@ -21,6 +22,7 @@ object UserRoleTable {
   val db = play.api.db.slick.DB
   val userRoles = TableQuery[UserRoleTable]
   val roles = TableQuery[RoleTable]
+  val userTable = TableQuery[UserTable]
 
   val roleMapping = Map("User" -> 1, "Turker" -> 2, "Researcher" -> 3, "Administrator" -> 4, "Owner" -> 5)
 
@@ -57,4 +59,13 @@ object UserRoleTable {
   def isResearcher(userId: UUID): Boolean = db.withSession { implicit session =>
     getRole(userId) == "Researcher"
   }
+
+  def getUsersByType(userType: String): Query[UserTable, DBUser, Seq] = {
+    val turkerUsers = for {
+      _roleIds <- userRoles
+      _roles <- roles if _roles.roleId === _roleIds.roleId && _roles.role === userType
+      _users <- userTable if _users.userId === _roleIds.userId
+    } yield _users
+    turkerUsers
+  }
 }

app/views/admin/index.scala.html

@@ -410,6 +410,7 @@ <h2>Total Time Spent Auditing</h2>
                     <div class="col-lg-12">
                         <span>Standard Deviation (all users): </span><span id="all-audittimes-std"></span><br>
                         <span>Standard Deviation (registered users): </span><span id="reg-audittimes-std"></span><br>
+                        <span>Standard Deviation (turker users): </span><span id="turker-audittimes-std"></span><br>
                         <span>Standard Deviation (anon users): </span><span id="anon-audittimes-std"></span><br>
                         <span>
                             Note: An anonymous user is defined as an IP address that is associated with at least one completed audit task.
@@ -429,6 +430,7 @@ <h2>Missions Completed By Users</h2>
                         <p>
                             <span>Standard Deviation (all users): </span><span id="missions-std"></span><br>
                             <span>Standard Deviation (registered users): </span><span id="reg-missions-std"></span><br>
+                            <span>Standard Deviation (turker users): </span><span id="turker-missions-std"></span><br>
                             <span>Standard Deviation (anon users): </span><span id="anon-missions-std"></span><br>
                             <span>
                                 Note: An anonymous user is defined as an IP address that is associated with at least one
@@ -446,6 +448,7 @@ <h2>Labels per User</h2>
                         <p>
                             <span>Standard Deviation (all users): </span><span id="all-labels-std"></span><br>
                             <span>Standard Deviation (registered users): </span><span id="reg-labels-std"></span><br>
+                            <span>Standard Deviation (turker users): </span><span id="turker-labels-std"></span><br>
                             <span>Standard Deviation (anon users): </span><span id="anon-labels-std"></span><br>
                             <span>
                                 Note: An anonymous user is defined as an IP address that is associated with at least one

app/views/admin/user.scala.html

@@ -57,8 +57,8 @@ <h1>General Info and Stats</h1>
                     <tr>
                         <th class="col-md-2">Current Neighborhood</th>
                         <td class="col-md-10">
-                            @RegionPropertyTable.neighborhoodName(UserCurrentRegionTable.currentRegion(UUID.fromString(user.get.userId)).get)
-                            (Region ID: @UserCurrentRegionTable.currentRegion(UUID.fromString(user.get.userId)))
+                            @UserCurrentRegionTable.currentRegion(UUID.fromString(user.get.userId)).map {_ => RegionPropertyTable.neighborhoodName(_)}.getOrElse("Unassigned")
+                            (Region ID: @UserCurrentRegionTable.currentRegion(UUID.fromString(user.get.userId)).map{_ => _}.getOrElse("NA"))
                         </td>
                     </tr>
                 </table>

build.sbt

@@ -4,7 +4,7 @@ import scalariform.formatter.preferences._
 
 name := """sidewalk-webpage"""
 
-version := "4.3.7"
+version := "4.3.8"
 
 scalaVersion := "2.10.4"
 

conf/evolutions/default/11.sql

@@ -0,0 +1,17 @@
+# --- !Ups
+UPDATE street_edge SET deleted = TRUE WHERE street_edge_id IN (
+  26762, 23790, 7738, 23705, 23845, 25214, 22855, 19178, 22918, 23785, 26758, 26763, 26773, 24838, 24431, 3919, 23850,
+  22846, 22946, 11224, 20517, 22931, 26765, 26764, 4710, 20564, 26769, 7430, 22775, 25543, 22711, 24692, 23801, 22972,
+  23802, 23792, 11223, 20526, 23851, 19177, 22971, 20525, 22772, 26772, 25171, 23386, 25332, 26770, 25213, 20520, 26774,
+  25325, 23841, 20562, 27027, 22822, 22768, 20523, 22851, 26767, 25331, 22920, 26777, 22930, 22911, 22820, 10615, 23385,
+  26757, 26766, 22767, 23846, 26778, 24691, 22847, 22845, 26761, 20866, 26785, 26930, 25538
+);
+
+# --- !Downs
+UPDATE street_edge SET deleted = FALSE WHERE street_edge_id IN (
+26762, 23790, 7738, 23705, 23845, 25214, 22855, 19178, 22918, 23785, 26758, 26763, 26773, 24838, 24431, 3919, 23850,
+22846, 22946, 11224, 20517, 22931, 26765, 26764, 4710, 20564, 26769, 7430, 22775, 25543, 22711, 24692, 23801, 22972,
+23802, 23792, 11223, 20526, 23851, 19177, 22971, 20525, 22772, 26772, 25171, 23386, 25332, 26770, 25213, 20520, 26774,
+25325, 23841, 20562, 27027, 22822, 22768, 20523, 22851, 26767, 25331, 22920, 26777, 22930, 22911, 22820, 10615, 23385,
+26757, 26766, 22767, 23846, 26778, 24691, 22847, 22845, 26761, 20866, 26785, 26930, 25538
+);

conf/routes

@@ -50,8 +50,10 @@ GET     /adminapi/labels/panoid                              @controllers.AdminC
 GET     /adminapi/label/:labelId                             @controllers.AdminController.getLabelData(labelId: Int)
 GET     /adminapi/labelCounts/registered                     @controllers.AdminController.getAllRegisteredUserLabelCounts
 GET     /adminapi/labelCounts/anonymous                      @controllers.AdminController.getAllAnonUserLabelCounts
+GET     /adminapi/labelCounts/turker                         @controllers.AdminController.getAllTurkerUserLabelCounts
 GET     /adminapi/audittimes                                 @controllers.AdminController.getAuditTimes()
 GET     /adminiapi/audittimesAnon                            @controllers.AdminController.getAnonAuditTimes()
+GET     /adminapi/audittimesTurker                           @controllers.AdminController.getTurkerAuditTimes()
 
 GET     /adminapi/webpageActivity                            @controllers.AdminController.getAllWebpageActivities
 GET     /adminapi/webpageActivity/:activity                  @controllers.AdminController.getWebpageActivities(activity: String)
@@ -106,6 +108,7 @@ GET     /isTurker                                            @controllers.UserPr
 GET     /userapi/labels                                      @controllers.UserProfileController.getSubmittedLabels(regionId: Option[Int] ?= None)
 GET     /userapi/labelCounts/all                             @controllers.UserProfileController.getAllLabelCounts
 GET     /userapi/completedMissionCounts/all                  @controllers.UserProfileController.getAllUserCompletedMissionCounts
+GET     /userapi/completedMissionCounts/turker               @controllers.UserProfileController.getTurkerCompletedMissionCounts
 
 POST    /userapi/logWebpageActivity                          @controllers.UserController.logWebpageActivity