FHIR Core Enhancements (#3587)

* Migrate to latest FHIR SDK Libraries

* Implement CPU async foreach extension method

* Refactor P2P data transer to execute in parallel

* Add content to test workflow APIs

* Revert Quest CI log level to Stacktrace

* Update sync jobs configuration

* Performance UX Improvements ⚡️

* Syncing in progress notifications

* Fix crash on Android 12+ devices

* Add Last sync duration info

* Add Foreground Service data sync permission

* Fix bug foreground service exception Android 14

.github/workflows/ci.yml

@@ -272,11 +272,7 @@ jobs:
           force-avd-creation: true
           emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           disable-animations: true
-          script: ./gradlew clean  -PlocalPropertiesFile=local.properties :quest:connectedOpensrpDebugAndroidTest --stacktrace -Pandroid.testInstrumentationRunnerArguments.notPackage=org.smartregister.fhircore.quest.performance
-
-      - name: Test UnitTest
-        run: ./gradlew clean  -PlocalPropertiesFile=local.properties :quest:testOpensrpDebugUnitTest --stacktrace
-        working-directory: android
+          script: ./gradlew clean  -PlocalPropertiesFile=local.properties :quest:fhircoreJacocoReport --stacktrace -Pandroid.testInstrumentationRunnerArguments.notPackage=org.smartregister.fhircore.quest.performance
 
       - name: Run Quest module unit and instrumentation tests and generate aggregated coverage report (Disabled)
         if: false

android/engine/build.gradle.kts

@@ -196,6 +196,7 @@ dependencies {
     exclude(group = "com.google.android.fhir", module = "engine")
     exclude(group = "org.smartregister", module = "engine")
     exclude(group = "com.github.ben-manes.caffeine")
+    exclude(group = "com.google.android.fhir", module = "knowledge")
   }
   api(libs.contrib.barcode) {
     isTransitive = true

android/engine/src/main/java/org/smartregister/fhircore/engine/configuration/ConfigurationRegistry.kt

@@ -34,8 +34,6 @@ import java.util.PropertyResourceBundle
 import java.util.ResourceBundle
 import javax.inject.Inject
 import javax.inject.Singleton
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withContext
 import kotlinx.serialization.json.Json
 import okhttp3.RequestBody.Companion.toRequestBody
@@ -100,7 +98,6 @@ constructor(
   val localizationHelper: LocalizationHelper by lazy { LocalizationHelper(this) }
   private val supportedFileExtensions = listOf("json", "properties")
   private var _isNonProxy = BuildConfig.IS_NON_PROXY_APK
-  private val mutex = Mutex()
 
   /**
    * Retrieve configuration for the provided [ConfigType]. The JSON retrieved from [configsJsonMap]
@@ -592,34 +589,30 @@ constructor(
    * Note
    */
   suspend fun <R : Resource> addOrUpdate(resource: R) {
-    withContext(dispatcherProvider.io()) {
-      try {
-        createOrUpdateRemote(resource)
-      } catch (sqlException: SQLException) {
-        Timber.e(sqlException)
-      }
+    try {
+      createOrUpdateRemote(resource)
+    } catch (sqlException: SQLException) {
+      Timber.e(sqlException)
+    }
 
-      /**
-       * Knowledge manager [MetadataResource]s install. Here we install all resources types of
-       * [MetadataResource] as per FHIR Spec.This supports future use cases as well
-       */
-      try {
-        if (resource is MetadataResource) {
-          mutex.withLock {
-            knowledgeManager.install(
-              KnowledgeManagerUtil.writeToFile(
-                context = context,
-                configService = configService,
-                metadataResource = resource,
-                subFilePath =
-                  "${KnowledgeManagerUtil.KNOWLEDGE_MANAGER_ASSETS_SUBFOLDER}/${resource.resourceType}/${resource.idElement.idPart}.json",
-              ),
-            )
-          }
-        }
-      } catch (exception: Exception) {
-        Timber.e(exception)
+    /**
+     * Knowledge manager [MetadataResource]s install. Here we install all resources types of
+     * [MetadataResource] as per FHIR Spec.This supports future use cases as well
+     */
+    try {
+      if (resource is MetadataResource) {
+        knowledgeManager.install(
+          KnowledgeManagerUtil.writeToFile(
+            context = context,
+            configService = configService,
+            metadataResource = resource,
+            subFilePath =
+              "${KnowledgeManagerUtil.KNOWLEDGE_MANAGER_ASSETS_SUBFOLDER}/${resource.resourceType}/${resource.idElement.idPart}.json",
+          ),
+        )
       }
+    } catch (exception: Exception) {
+      Timber.e(exception)
     }
   }
 
@@ -632,13 +625,11 @@ constructor(
    * @param resources vararg of resources
    */
   suspend fun createOrUpdateRemote(vararg resources: Resource) {
-    return withContext(dispatcherProvider.io()) {
-      resources.onEach {
-        it.updateLastUpdated()
-        it.generateMissingId()
-      }
-      fhirEngine.create(*resources, isLocalOnly = true)
+    resources.onEach {
+      it.updateLastUpdated()
+      it.generateMissingId()
     }
+    fhirEngine.create(*resources, isLocalOnly = true)
   }
 
   @VisibleForTesting fun isNonProxy(): Boolean = _isNonProxy
@@ -750,7 +741,7 @@ constructor(
                   it.system.contentEquals(organizationResourceTag?.tag?.system, ignoreCase = true)
                 }
                 ?.code
-            COUNT -> appConfig.remoteSyncPageSize.toString()
+            COUNT -> DEFAULT_COUNT.toString()
             else -> paramExpression
           }?.let { paramExpression?.replace(paramLiteral, it) }
 
@@ -801,7 +792,7 @@ constructor(
     const val MANIFEST_PROCESSOR_BATCH_SIZE = 20
     const val ORGANIZATION = "organization"
     const val TYPE_REFERENCE_DELIMITER = "/"
-    const val DEFAULT_COUNT = 200
+    const val DEFAULT_COUNT = 1000
     const val PAGINATION_NEXT = "next"
     const val RESOURCES_PATH = "resources/"
     const val SYNC_LOCATION_IDS = "_syncLocations"

android/engine/src/main/java/org/smartregister/fhircore/engine/configuration/app/ApplicationConfiguration.kt

@@ -19,6 +19,7 @@ package org.smartregister.fhircore.engine.configuration.app
 import kotlinx.serialization.Serializable
 import org.smartregister.fhircore.engine.configuration.ConfigType
 import org.smartregister.fhircore.engine.configuration.Configuration
+import org.smartregister.fhircore.engine.configuration.ConfigurationRegistry
 import org.smartregister.fhircore.engine.configuration.event.EventWorkflow
 import org.smartregister.fhircore.engine.domain.model.LauncherType
 import org.smartregister.fhircore.engine.util.extension.DEFAULT_FORMAT_SDF_DD_MM_YYYY
@@ -28,7 +29,7 @@ data class ApplicationConfiguration(
   override var appId: String,
   override var configType: String = ConfigType.Application.name,
   val appTitle: String = "",
-  val remoteSyncPageSize: Int = 100,
+  val remoteSyncPageSize: Int = ConfigurationRegistry.DEFAULT_COUNT,
   val languages: List<String> = listOf("en"),
   val useDarkTheme: Boolean = false,
   val syncInterval: Long = 15,

android/engine/src/main/java/org/smartregister/fhircore/engine/configuration/app/ConfigService.kt

@@ -34,6 +34,9 @@ interface ConfigService {
   /** Define a list of [ResourceTag] for the application. */
   fun defineResourceTags(): List<ResourceTag>
 
+  /** Return the App's launcher icon for use in notifications */
+  fun getLauncherIcon(): Int
+
   /**
    * Provide a list of [Coding] that represents [ResourceTag]. [Coding] can be directly appended to
    * a FHIR resource.

android/engine/src/main/java/org/smartregister/fhircore/engine/sync/AppSyncWorker.kt

@@ -16,20 +16,36 @@
 
 package org.smartregister.fhircore.engine.sync
 
+import android.app.Notification
+import android.app.NotificationChannel
+import android.app.NotificationManager
 import android.content.Context
+import android.content.pm.ServiceInfo
+import android.graphics.drawable.Icon
+import android.os.Build
+import androidx.core.app.NotificationCompat
 import androidx.hilt.work.HiltWorker
+import androidx.work.ForegroundInfo
 import androidx.work.WorkerParameters
 import com.google.android.fhir.FhirEngine
 import com.google.android.fhir.sync.AcceptLocalConflictResolver
 import com.google.android.fhir.sync.ConflictResolver
+import com.google.android.fhir.sync.CurrentSyncJobStatus
 import com.google.android.fhir.sync.DownloadWorkManager
 import com.google.android.fhir.sync.FhirSyncWorker
+import com.google.android.fhir.sync.SyncJobStatus
+import com.google.android.fhir.sync.SyncOperation
 import com.google.android.fhir.sync.upload.HttpCreateMethod
 import com.google.android.fhir.sync.upload.HttpUpdateMethod
 import com.google.android.fhir.sync.upload.UploadStrategy
+import com.ibm.icu.util.Calendar
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedInject
 import kotlinx.coroutines.runBlocking
+import org.smartregister.fhircore.engine.R
+import org.smartregister.fhircore.engine.configuration.app.ConfigService
+import org.smartregister.fhircore.engine.util.NotificationConstants
+import org.smartregister.fhircore.engine.util.SharedPreferenceKey
 
 @HiltWorker
 class AppSyncWorker
@@ -40,7 +56,14 @@ constructor(
   val syncListenerManager: SyncListenerManager,
   private val openSrpFhirEngine: FhirEngine,
   private val appTimeStampContext: AppTimeStampContext,
-) : FhirSyncWorker(appContext, workerParams) {
+  private val configService: ConfigService,
+) : FhirSyncWorker(appContext, workerParams), OnSyncListener {
+  private val notificationManager =
+    appContext.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+
+  init {
+    syncListenerManager.registerSyncListener(this)
+  }
 
   override fun getConflictResolver(): ConflictResolver = AcceptLocalConflictResolver
 
@@ -50,6 +73,26 @@ constructor(
       context = appTimeStampContext,
     )
 
+  override suspend fun doWork(): Result {
+    saveSyncStartTimestamp()
+    setForeground(getForegroundInfo())
+    return super.doWork()
+  }
+
+  private fun saveSyncStartTimestamp() {
+    syncListenerManager.sharedPreferencesHelper.write(
+      SharedPreferenceKey.SYNC_START_TIMESTAMP.name,
+      Calendar.getInstance().timeInMillis,
+    )
+  }
+
+  private fun saveSyncEndTimestamp() {
+    syncListenerManager.sharedPreferencesHelper.write(
+      SharedPreferenceKey.SYNC_END_TIMESTAMP.name,
+      Calendar.getInstance().timeInMillis,
+    )
+  }
+
   override fun getFhirEngine(): FhirEngine = openSrpFhirEngine
 
   override fun getUploadStrategy(): UploadStrategy =
@@ -59,4 +102,77 @@ constructor(
       squash = true,
       bundleSize = 500,
     )
+
+  override suspend fun getForegroundInfo(): ForegroundInfo {
+    val channel =
+      NotificationChannel(
+        NotificationConstants.ChannelId.DATA_SYNC,
+        NotificationConstants.ChannelName.DATA_SYNC,
+        NotificationManager.IMPORTANCE_LOW,
+      )
+    notificationManager.createNotificationChannel(channel)
+
+    val notification: Notification =
+      buildNotification(progress = 0, isSyncUpload = false, isInitial = true)
+
+    val foregroundInfo =
+      if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+        ForegroundInfo(
+          NotificationConstants.NotificationId.DATA_SYNC,
+          notification,
+          ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC,
+        )
+      } else {
+        ForegroundInfo(NotificationConstants.NotificationId.DATA_SYNC, notification)
+      }
+    return foregroundInfo
+  }
+
+  private fun getSyncProgress(completed: Int, total: Int) =
+    completed * 100 / if (total > 0) total else 1
+
+  override fun onSync(syncJobStatus: CurrentSyncJobStatus) {
+    when (syncJobStatus) {
+      is CurrentSyncJobStatus.Running -> {
+        if (syncJobStatus.inProgressSyncJob is SyncJobStatus.InProgress) {
+          val inProgressSyncJob = syncJobStatus.inProgressSyncJob as SyncJobStatus.InProgress
+          val isSyncUpload = inProgressSyncJob.syncOperation == SyncOperation.UPLOAD
+          val progressPercentage =
+            getSyncProgress(inProgressSyncJob.completed, inProgressSyncJob.total)
+          updateNotificationProgress(progress = progressPercentage, isSyncUpload = isSyncUpload)
+        }
+      }
+      is CurrentSyncJobStatus.Succeeded -> saveSyncEndTimestamp()
+      else -> {}
+    }
+  }
+
+  private fun buildNotification(
+    progress: Int,
+    isSyncUpload: Boolean,
+    isInitial: Boolean,
+  ): Notification {
+    return NotificationCompat.Builder(applicationContext, NotificationConstants.ChannelId.DATA_SYNC)
+      .setContentTitle(
+        applicationContext.getString(
+          if (isInitial) {
+            R.string.syncing_initiated
+          } else if (isSyncUpload) R.string.syncing_up else R.string.syncing_down,
+        ),
+      )
+      .setSmallIcon(R.drawable.ic_opensrp_small_logo)
+      .setLargeIcon(Icon.createWithResource(applicationContext, configService.getLauncherIcon()))
+      .setContentText(applicationContext.getString(R.string.percentage_progress, progress))
+      .setProgress(100, progress, progress == 0)
+      .setOngoing(true)
+      .build()
+  }
+
+  private fun updateNotificationProgress(progress: Int, isSyncUpload: Boolean) {
+    val notificationManager =
+      applicationContext.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+    val notification =
+      buildNotification(progress = progress, isSyncUpload = isSyncUpload, isInitial = false)
+    notificationManager.notify(NotificationConstants.NotificationId.DATA_SYNC, notification)
+  }
 }

android/engine/src/main/java/org/smartregister/fhircore/engine/sync/CustomSyncWorker.kt

@@ -49,7 +49,7 @@ constructor(
           val (resourceSearchParams, _) = loadResourceSearchParams()
           Timber.i("Custom resource sync parameters $resourceSearchParams")
           resourceSearchParams
-            .asSequence()
+            .asIterable()
             .filter { it.value.isNotEmpty() }
             .map { "${it.key}?${it.value.concatParams()}" }
             .forEach { url ->

android/engine/src/main/java/org/smartregister/fhircore/engine/sync/OpenSrpDownloadManager.kt

@@ -20,6 +20,7 @@ import com.google.android.fhir.sync.DownloadWorkManager
 import com.google.android.fhir.sync.download.DownloadRequest
 import com.google.android.fhir.sync.download.ResourceParamsBasedDownloadWorkManager
 import com.google.android.fhir.sync.download.ResourceSearchParams
+import com.google.android.fhir.sync.download.UrlDownloadRequest
 import org.hl7.fhir.r4.model.Resource
 import org.hl7.fhir.r4.model.ResourceType
 import org.smartregister.fhircore.engine.util.extension.updateLastUpdated
@@ -32,7 +33,12 @@ class OpenSrpDownloadManager(
   private val downloadWorkManager =
     ResourceParamsBasedDownloadWorkManager(resourceSearchParams, context)
 
-  override suspend fun getNextRequest(): DownloadRequest? = downloadWorkManager.getNextRequest()
+  override suspend fun getNextRequest(): DownloadRequest? =
+    downloadWorkManager.getNextRequest().apply {
+      if (this is UrlDownloadRequest) {
+        url.replace("_pretty=true", "_pretty=false")
+      }
+    }
 
   override suspend fun getSummaryRequestUrls(): Map<ResourceType, String> =
     downloadWorkManager.getSummaryRequestUrls()