Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: update local asset driver #144

Merged
merged 3 commits into from
Jun 13, 2024
Merged

feat: update local asset driver #144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
98c369b
feat: update local asset driver
Rybasher Apr 23, 2024
e98bed6
Merge branch 'dev' into feat/update-local-asset-driver
Rybasher May 4, 2024
c1d7021
Merge branch 'dev' into feat/update-local-asset-driver
Rybasher Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion mirror-web-server/src/app.module.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,17 @@ if (
console.log('Added AnalyticsModule')
}

if (process.env.ASSET_STORAGE_DRIVER === 'LOCAL') {
if (
!process.env.ASSET_STORAGE_DRIVER ||
process.env.ASSET_STORAGE_DRIVER === 'LOCAL'
) {
console.log(
'Using local storage, path: ',
join(__dirname, '../', 'localStorage')
)
imports.push(
ServeStaticModule.forRoot({
serveRoot: '/assets-storage',
rootPath: join(__dirname, '../', 'localStorage')
})
)
Expand Down
16 changes: 16 additions & 0 deletions mirror-web-server/src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,22 @@ async function bootstrapMirrorWebServer() {
})
app.useWebSocketAdapter(new WsAdapter(app))

// asset storage driver env validation
if (
process.env.ASSET_STORAGE_DRIVER === 'GCP' &&
!process.env.GCS_BUCKET_PUBLIC
) {
throw new Error('GCS_BUCKET_PUBLIC is required when using GCP storage')
}

if (
(!process.env.ASSET_STORAGE_DRIVER ||
process.env.ASSET_STORAGE_DRIVER === 'LOCAL') &&
!process.env.ASSET_STORAGE_URL
) {
throw new Error('ASSET_STORAGE_URL is required when using LOCAL storage')
}

// Important: This *only* sets up validation pipes for HTTP handlers, NOT Websockets. See the notice here: https://docs.nestjs.com/pipes#global-scoped-pipes
app.useGlobalPipes(
new ValidationPipe({
Expand Down
29 changes: 17 additions & 12 deletions mirror-web-server/src/space/space-godot-server.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,23 +82,28 @@ export class SpaceGodotServerController {
try {
const remoteRelativePath = `space/${id}/terrain/voxels.dat`

if (process.env.ASSET_STORAGE_DRIVER === 'LOCAL') {
await this.fileUploadService.uploadFileLocal(file, remoteRelativePath)
if (process.env.ASSET_STORAGE_DRIVER === 'GCP') {
await this.fileUploadService.streamFile(
process.env.GCS_BUCKET_PUBLIC,
remoteRelativePath,
file,
'publicRead'
)
return {
success: true,
publicUrl: `${process.env.ASSET_STORAGE_URL}/${remoteRelativePath}`
publicUrl: `${process.env.GCP_BASE_PUBLIC_URL}/${remoteRelativePath}`
}
}

await this.fileUploadService.streamFile(
process.env.GCS_BUCKET_PUBLIC,
remoteRelativePath,
file,
'publicRead'
)
return {
success: true,
publicUrl: `${process.env.GCP_BASE_PUBLIC_URL}/${remoteRelativePath}`
if (
!process.env.ASSET_STORAGE_DRIVER ||
process.env.ASSET_STORAGE_DRIVER === 'LOCAL'
) {
await this.fileUploadService.uploadFileLocal(file, remoteRelativePath)
return {
success: true,
publicUrl: `${process.env.ASSET_STORAGE_URL}/${remoteRelativePath}`
}
}
} catch (e) {
this.logger.error(e?.message, e, SpaceGodotServerController.name)
Expand Down
35 changes: 21 additions & 14 deletions mirror-web-server/src/space/space.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1045,7 +1045,24 @@ export class SpaceService implements IRoleConsumer {
try {
const remoteRelativePath = `space/${spaceId}/terrain/voxels.dat`

if (process.env.ASSET_STORAGE_DRIVER === 'LOCAL') {
if (process.env.ASSET_STORAGE_DRIVER === 'GCP') {
await this.fileUploadService.streamData(
process.env.GCS_BUCKET_PUBLIC,
remoteRelativePath,
'application/octet-stream',
Buffer.from(new Uint8Array()),
'publicRead'
)
return {
success: true,
publicUrl: `${process.env.GCP_BASE_PUBLIC_URL}/${remoteRelativePath}`
}
}

if (
!process.env.ASSET_STORAGE_DRIVER ||
process.env.ASSET_STORAGE_DRIVER === 'LOCAL'
) {
// Create a fake file object to pass to the local file upload method
const file: Express.Multer.File = {
fieldname: '',
Expand All @@ -1054,29 +1071,19 @@ export class SpaceService implements IRoleConsumer {
mimetype: 'application/octet-stream',
size: 0,
destination: '',
filename: '',
filename: 'voxels.dat',
path: '',
buffer: Buffer.from(new Uint8Array()),
stream: null // Add the 'stream' property
}

await this.fileUploadService.uploadFileLocal(file, remoteRelativePath)

return {
success: true,
publicUrl: `${process.env.ASSET_STORAGE_URL}/${remoteRelativePath}`
}
}

await this.fileUploadService.streamData(
process.env.GCS_BUCKET_PUBLIC,
remoteRelativePath,
'application/octet-stream',
Buffer.from(new Uint8Array()),
'publicRead'
)
return {
success: true,
publicUrl: `${process.env.GCP_BASE_PUBLIC_URL}/${remoteRelativePath}`
}
} catch (e) {
this.logger.error(e?.message, e, SpaceService.name)
throw new InternalServerErrorException('Error uploading terrain data')
Expand Down
18 changes: 12 additions & 6 deletions mirror-web-server/src/terrain/terrain.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,14 +87,20 @@ export class TerrainService {
const fromPath = `space/${fromId}/terrain/voxels.dat`
const toPath = `space/${toId}/terrain/voxels.dat`
try {
if (process.env.ASSET_STORAGE_DRIVER === 'LOCAL') {
if (process.env.ASSET_STORAGE_DRIVER === 'GCP') {
await this.fileUploadService.copyFileInBucket(
process.env.GCS_BUCKET_PUBLIC,
fromPath,
toPath
)
}

if (
!process.env.ASSET_STORAGE_DRIVER ||
process.env.ASSET_STORAGE_DRIVER === 'LOCAL'
) {
return await this.fileUploadService.copyFileLocal(fromPath, toPath)
}
await this.fileUploadService.copyFileInBucket(
process.env.GCS_BUCKET_PUBLIC,
fromPath,
toPath
)
} catch (error: any) {
/** When no voxel exists, one will be created by godot */
const message: string = error?.message || 'No error message provided'
Expand Down
52 changes: 31 additions & 21 deletions mirror-web-server/src/util/file-upload/file-upload.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,24 +59,29 @@ export class FileUploadService implements FileUploadInterface {
file.mimetype
)}`

if (process.env.ASSET_STORAGE_DRIVER === 'GCP') {
await this.streamFile(
process.env.GCS_BUCKET_PUBLIC,
pathWithFileType,
file,
'publicRead'
)
return {
publicUrl: `${process.env.GCP_BASE_PUBLIC_URL}/${pathWithFileType}`
}
}

// If we're using local asset storage, we'll just save the file to the local storage
if (process.env.ASSET_STORAGE_DRIVER === 'LOCAL') {
if (
!process.env.ASSET_STORAGE_DRIVER ||
process.env.ASSET_STORAGE_DRIVER === 'LOCAL'
) {
console.log('Uploading file to local storage')
await this.uploadFileLocal(file, pathWithFileType)
return {
publicUrl: process.env.ASSET_STORAGE_URL + pathWithFileType
}
}

await this.streamFile(
process.env.GCS_BUCKET_PUBLIC,
pathWithFileType,
file,
'publicRead'
)
return {
publicUrl: `${process.env.GCP_BASE_PUBLIC_URL}/${pathWithFileType}`
}
} catch (error: any) {
const message: string = error?.message
throw new HttpException(`File upload error: ${message}`, 400)
Expand All @@ -95,20 +100,25 @@ export class FileUploadService implements FileUploadInterface {
file.mimetype
)}`

if (process.env.ASSET_STORAGE_DRIVER === 'GCP') {
await this.streamFile(
process.env.GCS_BUCKET,
pathWithFileType,
file,
'private'
)

return { relativePath: pathWithFileType }
}

// If we're using local asset storage, we'll just save the file to the local storage
if (process.env.ASSET_STORAGE_DRIVER === 'LOCAL') {
if (
!process.env.ASSET_STORAGE_DRIVER ||
process.env.ASSET_STORAGE_DRIVER === 'LOCAL'
) {
await this.uploadFileLocal(file, pathWithFileType)
return { relativePath: pathWithFileType }
}

await this.streamFile(
process.env.GCS_BUCKET,
pathWithFileType,
file,
'private'
)

return { relativePath: pathWithFileType }
} catch (error: any) {
const message: string = error?.message
throw new HttpException(`File upload error: ${message}`, 400)
Expand Down
Loading