Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

테스트용 bulk insert를 추가한다 #359

Merged
merged 1 commit into from
Aug 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package com.dclass.backend.common

import com.dclass.backend.domain.blacklist.Blacklist
import com.dclass.backend.domain.post.Post
import jakarta.transaction.Transactional
import org.springframework.jdbc.core.BatchPreparedStatementSetter
import org.springframework.jdbc.core.JdbcTemplate
import org.springframework.stereotype.Repository
import java.sql.PreparedStatement

@Repository
class BulkInsertRepository(
private val jdbcTemplate: JdbcTemplate
) {

@Transactional
fun bulkInsert(values: List<Post>): Long? {
val sql =
"""INSERT INTO post (community_id, content, created_date_time, deleted, is_anonymous, is_question, modified_date_time, comment_reply_count, like_count, scrap_count, title, user_id, version,id)
|VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""".trimMargin()
jdbcTemplate.batchUpdate(
sql,
object : BatchPreparedStatementSetter {
override fun setValues(ps: PreparedStatement, i: Int) {
val post = values[i]
ps.setLong(1, post.communityId)
ps.setString(2, post.content)
ps.setObject(3, post.createdDateTime)
ps.setBoolean(4, false)
ps.setBoolean(5, post.isAnonymous)
ps.setBoolean(6, post.isQuestion)
ps.setObject(7, post.modifiedDateTime)
ps.setInt(8, post.postCount.commentReplyCount)
ps.setInt(9, post.postCount.likeCount)
ps.setInt(10, post.postCount.scrapCount)
ps.setString(11, post.title)
ps.setLong(12, post.userId)
ps.setLong(13, post.version)
ps.setLong(14, post.id)
}

override fun getBatchSize(): Int {
return values.size
}
},
)
return jdbcTemplate.queryForObject("SELECT LAST_INSERT_ID()", Long::class.java)!!
}

@Transactional
fun bulkBlacklistInsert(values: List<Blacklist>): Long? {
val sql =
"""INSERT INTO blacklist (deleted, id, invalid_refresh_token)
|VALUES (?, ?, ?)""".trimMargin()
jdbcTemplate.batchUpdate(
sql,
object : BatchPreparedStatementSetter {
override fun setValues(ps: PreparedStatement, i: Int) {
val blacklists = values[i]
ps.setBoolean(1, false)
ps.setLong(2, blacklists.id)
ps.setString(3, blacklists.invalidRefreshToken)
}

override fun getBatchSize(): Int {
return values.size
}
},
)
return jdbcTemplate.queryForObject("SELECT LAST_INSERT_ID()", Long::class.java)!!
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package com.dclass.backend.config
import com.dclass.backend.application.CommentService
import com.dclass.backend.application.PostService
import com.dclass.backend.application.ReplyService
import com.dclass.backend.common.BulkInsertRepository
import com.dclass.backend.domain.belong.Belong
import com.dclass.backend.domain.belong.BelongRepository
import com.dclass.backend.domain.blacklist.Blacklist
import com.dclass.backend.domain.comment.Comment
import com.dclass.backend.domain.comment.CommentRepository
import com.dclass.backend.domain.community.Community
Expand All @@ -20,6 +22,8 @@ import com.dclass.backend.domain.user.University
import com.dclass.backend.domain.user.UniversityRepository
import com.dclass.backend.domain.user.User
import com.dclass.backend.domain.user.UserRepository
import com.dclass.backend.security.JwtTokenProvider
import jakarta.persistence.EntityManager
import jakarta.transaction.Transactional
import org.springframework.boot.CommandLineRunner
import org.springframework.context.annotation.Profile
Expand All @@ -31,6 +35,7 @@ import java.time.LocalDateTime
@Component
class DatabaseInitializer(
private val database: Database,
private val entityManager: EntityManager,
private val universityRepository: UniversityRepository,
private val departmentRepository: DepartmentRepository,
private val communityRepository: CommunityRepository,
Expand All @@ -42,8 +47,20 @@ class DatabaseInitializer(
private val commentService: CommentService,
private val replyService: ReplyService,
private val postService: PostService,
private val bulkInsertRepository: BulkInsertRepository,
) : CommandLineRunner {

private fun prev(){
entityManager.createNativeQuery("set foreign_key_checks = 0").executeUpdate()
entityManager.createNativeQuery("set autocommit = 0").executeUpdate()
}

private fun nxt(){
entityManager.createNativeQuery("commit").executeUpdate()
entityManager.createNativeQuery("set foreign_key_checks = 1").executeUpdate()
entityManager.createNativeQuery("set autocommit = 1").executeUpdate()
}

override fun run(vararg args: String) {
cleanUp()
populate()
Expand All @@ -58,6 +75,7 @@ class DatabaseInitializer(
populateDepartment()
populateCommunity()
populateUser()
// populateBlacklist()
// populateDummyPosts()
// populateDummyComments()
// populateDummyReplies()
Expand Down Expand Up @@ -775,28 +793,58 @@ class DatabaseInitializer(
val communityIds = (445L..450L).toList()

val dummyPosts = mutableListOf<Post>()

val batchSize = 10000
var number = 1

for (user in users) {
for (i in 1..10) {
for (i in 1..100000) {
val communityId = communityIds.random()
val community = communityRepository.findById(communityId).get()
val post = createDummyPost(user, community, number)
dummyPosts.add(post)
number++


if (dummyPosts.size >= batchSize) {
prev()
bulkInsertRepository.bulkInsert(dummyPosts)
nxt()
dummyPosts.clear()
}
}
}
}

private fun populateBlacklist(){
val blacklists = mutableListOf<Blacklist>()
var number = 1L
val batchSize = 100000
val totalSize = 100000 * 30
for(i in 1..totalSize){
val blacklist= Blacklist(invalidRefreshToken = generateRandomString(100),id=i.toLong())
blacklists.add(blacklist)
if(blacklists.size >= batchSize){
prev()
bulkInsertRepository.bulkBlacklistInsert(blacklists)
nxt()
blacklists.clear()
}
}
}

postRepository.saveAll(dummyPosts)
fun generateRandomString(length: Int): String {
val chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
return (1..length)
.map { chars.random() }
.joinToString("")
}

private fun createDummyPost(user: User, community: Community, number: Int): Post {
val title = "Dummy Post Title $number"
val content = "Dummy Post Content"
val createdDateTime = LocalDateTime.now()

return Post(user.id, community.id, title, content, PostLikes(), createdDateTime = createdDateTime)
return Post(user.id, community.id, title, content, PostLikes(), createdDateTime = createdDateTime,id=number.toLong())
}

private fun populateDummyComments() {
Expand Down
3 changes: 2 additions & 1 deletion backend/src/main/resources/application-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ spring:
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://localhost:3306/dclass?serverTimezone=UTC
# url: jdbc:mysql://localhost:3306/dclass?rewriteBatchedStatements=true&profileSQL=true&logger=Slf4JLogger&maxQuerySizeToLog=999999
username: dclass-user
password: password

Expand Down Expand Up @@ -39,4 +40,4 @@ springdoc:
disable-swagger-default-url: true
display-request-duration: true
operations-sorter: alpha
---
---
Loading