Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Endringer på metrics #171

Merged
merged 2 commits into from
Feb 11, 2024
Merged

Endringer på metrics #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a09980b
Endringer på metrics
nils-arne Feb 7, 2024
2fd6b19
Endringer etter PR
nils-arne Feb 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 13 additions & 9 deletions ...erer/src/main/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/ArchiverService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,13 @@ package no.nav.soknad.arkivering.soknadsarkiverer.service
import kotlinx.coroutines.*
import no.nav.soknad.arkivering.avroschemas.InnsendingMetrics
import no.nav.soknad.arkivering.avroschemas.Soknadarkivschema
import no.nav.soknad.arkivering.soknadsarkiverer.Constants
import no.nav.soknad.arkivering.soknadsarkiverer.config.ArchivingException
import no.nav.soknad.arkivering.soknadsarkiverer.config.ShuttingDownException
import no.nav.soknad.arkivering.soknadsarkiverer.kafka.KafkaPublisher
import no.nav.soknad.arkivering.soknadsarkiverer.service.arkivservice.JournalpostClientInterface
import no.nav.soknad.arkivering.soknadsarkiverer.service.fileservice.*
import no.nav.soknad.arkivering.soknadsarkiverer.supervision.ArchivingMetrics
import no.nav.soknad.arkivering.soknadsfillager.model.FileData
import org.slf4j.LoggerFactory
import org.slf4j.MDC
import org.springframework.stereotype.Service
import java.io.PrintWriter
import java.io.StringWriter
Expand All @@ -29,7 +26,7 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
try {
val startTime = System.currentTimeMillis()
val journalpostId = journalpostClient.opprettJournalpost(key, data, files)
createMetric(key, "send files to archive", startTime)
createMetricAndPublishOnKafka(key, "send files to archive", startTime)
logger.info("$key: Opprettet journalpostId=$journalpostId for behandlingsid=${data.behandlingsid}")
createMessage(key, "**Archiving: OK. journalpostId=$journalpostId")

Expand All @@ -47,7 +44,13 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
return try {
val startTime = System.currentTimeMillis()
val files = makeParallelCallsToFetchFiles(key, data)
createMetric(key, "get files from filestorage", startTime)

createMetricAndPublishOnKafka(key, "get files from filestorage", startTime)
files.filter{it.content != null}.forEach {
metrics.setFileFetchSizeHistogram(it.content!!.size.toDouble(), data.arkivtema)
metrics.setFileFetchSize(it.content.size.toDouble())
}

files

} catch (e: ShuttingDownException) {
Expand Down Expand Up @@ -79,10 +82,11 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
val okResponse = responses.firstOrNull { it.status == ResponseStatus.Ok.value }
if (okResponse != null) {
metrics.incGetFilestorageSuccesses()
if (okResponse.files != null)
if (okResponse.files != null) {
return okResponse.files
else
} else {
return listOf()
}
}

val deletedResponse = responses.firstOrNull { it.status == "deleted" }
Expand All @@ -109,7 +113,7 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
try {
val startTime = System.currentTimeMillis()
filestorageService.deleteFilesFromFilestorage(key, data)
createMetric(key, "delete files from filestorage", startTime)
createMetricAndPublishOnKafka(key, "delete files from filestorage", startTime)
createMessage(key, "ok")

} catch (e: ShuttingDownException) {
Expand All @@ -127,7 +131,7 @@ class ArchiverService(private val filestorageService: FileserviceInterface,
kafkaPublisher.putMessageOnTopic(key, message)
}

private fun createMetric(key: String, message: String, startTime: Long) {
private fun createMetricAndPublishOnKafka(key: String, message: String, startTime: Long) {
val duration = System.currentTimeMillis() - startTime
val metrics = InnsendingMetrics("soknadsarkiverer", message, startTime, duration)
kafkaPublisher.putMetricOnTopic(key, metrics)
Expand Down
2 changes: 1 addition & 1 deletion ...erer/src/main/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/TaskListService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ open class TaskListService(
MDC.clear()
metrics.endTimer(timer)
metrics.endHistogramTimer(histogram)
metrics.numberOfAttachmentHistogramSet(
metrics.setNumberOfAttachmentHistogram(
soknadarkivschema.mottatteDokumenter.size.toDouble(),
soknadarkivschema.arkivtema
)
Expand Down
3 changes: 1 addition & 2 deletions ...otlin/no/nav/soknad/arkivering/soknadsarkiverer/service/arkivservice/JournalpostClient.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import no.nav.soknad.arkivering.soknadsarkiverer.service.arkivservice.api.Oppret
import no.nav.soknad.arkivering.soknadsarkiverer.service.arkivservice.converter.createOpprettJournalpostRequest
import no.nav.soknad.arkivering.soknadsarkiverer.service.fileservice.FileInfo
import no.nav.soknad.arkivering.soknadsarkiverer.supervision.ArchivingMetrics
import no.nav.soknad.arkivering.soknadsfillager.model.FileData
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.beans.factory.annotation.Value
Expand All @@ -32,7 +31,7 @@ class JournalpostClient(@Value("\${joark.host}") private val joarkHost: String,
val bidClient: WebClient = webClient.mutate().defaultHeader(NAV_CONSUMER_ID, "dialogstyring-bidrag").build()

override fun opprettJournalpost(key: String, soknadarkivschema: Soknadarkivschema, attachedFiles: List<FileInfo>): String {
val timer = metrics.joarkLatencyStart()
val timer = metrics.startJoarkLatency()
try {
logger.info("$key: About to create journalpost for behandlingsId: '${soknadarkivschema.behandlingsid}'")
val request = createOpprettJournalpostRequest(soknadarkivschema, attachedFiles)
Expand Down
49 changes: 44 additions & 5 deletions ...src/main/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/supervision/ArchivingMetrics.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
private val SUMMARY_FILESTORAGE_DEL_LATENCY = "latency_filestorage_del_operations"
private val SUMMARY_FILESTORAGE_DEL_LATENCY_DESC = "Latency for deleting from filestorage"

private val SUMMARY_FILE_FETCH_SIZE = "file_fetch_size"
private val SUMMARY_FILE_FETCH_SIZE_DESC = "Size of fetched file"
private val HISTOGRAM_FILE_FETCH_SIZE = "file_fetch_size_distribution"
private val HISTOGRAM_FILE_FETCH_SIZE_DESC = "Distribution of sizes of fetched files"

private val COUNTER_JOARK_SUCCESS = "counter_joark_success"
private val COUNTER_JOARK_SUCCESS_DESC = "Number of successes when sending to Joark"

Expand All @@ -67,10 +72,12 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
private val filestorageDelErrorCounter: Counter = registerCounter(COUNTER_FILESTORAGE_DEL_ERROR, COUNTER_FILESTORAGE_DEL_ERROR_DESC)
private val filestorageGetLatencySummary = registerSummary(SUMMARY_FILESTORAGE_GET_LATENCY, SUMMARY_FILESTORAGE_GET_LATENCY_DESC)
private val filestorageDelLatencySummary = registerSummary(SUMMARY_FILESTORAGE_DEL_LATENCY, SUMMARY_FILESTORAGE_DEL_LATENCY_DESC)
private val filefetchSizeSummary = registerSummary(SUMMARY_FILE_FETCH_SIZE, SUMMARY_FILE_FETCH_SIZE_DESC)
private val filefetchSizeHistogram = registerFileSizeHistogram(HISTOGRAM_FILE_FETCH_SIZE, HISTOGRAM_FILE_FETCH_SIZE_DESC)
private val joarkSuccessCounter: Counter = registerCounter(COUNTER_JOARK_SUCCESS, COUNTER_JOARK_SUCCESS_DESC)
private val joarkErrorCounter: Counter = registerCounter(COUNTER_JOARK_ERROR, COUNTER_JOARK_ERROR_DESC)
private val joarkLatencySummary = registerSummary(SUMMARY_JOARK_LATENCY, SUMMARY_JOARK_LATENCY_DESC)
private val archivingLatencyHistogram = registerHistogram(HISTOGRAM_ARCHIVING_LATENCY, HISTORGRAM_ARCHIVING_LATENCY_DESC)
private val archivingLatencyHistogram = registerLatencyHistogram(HISTOGRAM_ARCHIVING_LATENCY, HISTORGRAM_ARCHIVING_LATENCY_DESC)

private val HISTOGRAM_ATTACHMENT_NUMBER = "histogram_attachment_number"
private val HISTOGRAM_ATTACHMENT_NUMBER_DESC = "Histogram for number of attachment per application"
Expand Down Expand Up @@ -106,16 +113,41 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
.labelNames(APP_LABEL)
.register(registry)

private fun registerHistogram(name: String, help: String): Histogram =
private fun registerLatencyHistogram(name: String, help: String): Histogram =
Histogram
.build()
.namespace(SOKNAD_NAMESPACE)
.name(name)
.help(help)
.buckets(0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 100.2)
.buckets(0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 100.2, 240.0)
.labelNames(TEMA_LABEL)
.register(registry)

private fun registerFileSizeHistogram(name: String, help: String): Histogram {
val kB = 1024.0
val mB = kB * kB
return Histogram
.build()
.namespace(SOKNAD_NAMESPACE)
.name(name)
.help(help)
.buckets(
kB,
10 * kB,
50 * kB,
100 * kB,
500 * kB,
mB,
2 * mB,
5 * mB,
25 * mB,
50 * mB,
150 * mB
)
.labelNames(TEMA_LABEL)
.register(registry)
}

private fun registerAttachmentNumberHistogram(name: String, help: String): Histogram =
Histogram
.build()
Expand Down Expand Up @@ -156,9 +188,15 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
fun archivingLatencyStart(): Summary.Timer = archivingLatencySummary.labels(APP).startTimer()
fun filestorageGetLatencyStart(): Summary.Timer = filestorageGetLatencySummary.labels(APP).startTimer()
fun filestorageDelLatencyStart(): Summary.Timer = filestorageDelLatencySummary.labels(APP).startTimer()
fun joarkLatencyStart(): Summary.Timer = joarkLatencySummary.labels(APP).startTimer()
fun startJoarkLatency(): Summary.Timer = joarkLatencySummary.labels(APP).startTimer()
fun getJoarkLatency(): Summary.Child.Value = joarkLatencySummary.labels(APP).get()
fun archivingLatencyHistogramStart(tema: String): Histogram.Timer = archivingLatencyHistogram.labels(tema).startTimer()
fun numberOfAttachmentHistogramSet(number: Double, tema: String) = numberOfAttachmentHistogram.labels(tema).observe(number)
fun setNumberOfAttachmentHistogram(number: Double, tema: String) = numberOfAttachmentHistogram.labels(tema).observe(number)
fun getNumberOfAttachmentHistogram(tema: String) = numberOfAttachmentHistogram.labels(tema).get()
fun setFileFetchSize(size: Double) = filefetchSizeSummary.labels(APP).observe(size)
fun getFileFetchSize() = filefetchSizeSummary.labels(APP).get()
fun setFileFetchSizeHistogram(size: Double, tema: String) = filefetchSizeHistogram.labels(tema).observe(size)
fun getFileFetchSizeHistogram(tema: String) = filefetchSizeHistogram.labels(tema).get()

fun endTimer(timer: Summary.Timer) {
timer.observeDuration()
Expand All @@ -184,5 +222,6 @@ class ArchivingMetrics(private val registry: CollectorRegistry) {
registry.unregister(taskGauge)
registry.unregister(upOrDownGauge)
registry.unregister(numberOfAttachmentHistogram)
registry.unregister(filefetchSizeSummary)
}
}
44 changes: 38 additions & 6 deletions ...src/test/kotlin/no/nav/soknad/arkivering/soknadsarkiverer/service/ArchiverServiceTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@ package no.nav.soknad.arkivering.soknadsarkiverer.service

import io.mockk.*
import io.prometheus.client.CollectorRegistry
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.*
import no.nav.soknad.arkivering.soknadsarkiverer.SoknadsarkivererApplication
import no.nav.soknad.arkivering.soknadsarkiverer.config.ArchivingException
import no.nav.soknad.arkivering.soknadsarkiverer.kafka.KafkaPublisher
import no.nav.soknad.arkivering.soknadsarkiverer.service.arkivservice.JournalpostClientInterface
Expand All @@ -18,11 +17,15 @@ import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import java.time.OffsetDateTime.now
import java.util.*

class ArchiverServiceTests {

private lateinit var metrics: ArchivingMetrics

private val filestorage = mockk<FilestorageService>().also {
every {
it.getFilesFromFilestorage(any(), any()) } returns FetchFileResponse(status = "ok",
Expand Down Expand Up @@ -68,8 +71,6 @@ class ArchiverServiceTests {
private val journalpostClient = mockk<JournalpostClientInterface>().also {
every { it.opprettJournalpost(any(), any(), any()) } returns UUID.randomUUID().toString()
}
@Autowired
private lateinit var metrics: ArchivingMetrics

private val kafkaPublisher = mockk<KafkaPublisher>().also {
every { it.putMetricOnTopic(any(), any(), any()) } just Runs
Expand Down Expand Up @@ -104,6 +105,38 @@ class ArchiverServiceTests {
private val journalpostClient2 = mockk<JournalpostClientInterface>().also {
every { it.opprettJournalpost(any(), any(), capture(filer)) } returns UUID.randomUUID().toString()
}

@Test
fun `Fetch file metrics test`() {
archiverService = ArchiverService(filestorageNotFound, innsendingApi, journalpostClient2, metrics, kafkaPublisher)
val key = UUID.randomUUID().toString()
val tema = "AAP"
val soknadschema =
createSoknadarkivschema(behandlingsId = key,
tema = tema,
fileIds = listOf(
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
UUID.randomUUID().toString()
))

runBlocking {
archiverService.fetchFiles(key, soknadschema)

val fetchObservation = metrics.getFileFetchSize()
assertTrue(fetchObservation != null)
assertEquals(7.0, fetchObservation.quantiles[0.99]!!)
val fetchFileHistogram = metrics.getFileFetchSizeHistogram(tema)
assertTrue(fetchFileHistogram != null)
assertEquals("content".length.toDouble(), fetchFileHistogram.sum)
}
}

@Test
fun `Archiving succeeds when all is up and running`() {
archiverService = ArchiverService(filestorageNotFound, innsendingApi, journalpostClient2, metrics, kafkaPublisher)
Expand All @@ -118,7 +151,6 @@ class ArchiverServiceTests {
verify(exactly = 1) { journalpostClient2.opprettJournalpost(eq(key), eq(soknadschema), any()) }
assertTrue(filer.isCaptured)
assertEquals(soknadschema.mottatteDokumenter.first().mottatteVarianter.size, filer.captured.size)

}

}
Expand Down
Loading
Loading