Merge pull request #961 from funmusicplace/various-fixes

fix: some improvements to file moving script
funmusicplace · Jan 9, 2025 · df9b9dd · df9b9dd
2 parents 764dd95 + e440778
commit df9b9dd
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 21 deletions.
diff --git a/src/jobs/generate-album.ts b/src/jobs/generate-album.ts
@@ -206,7 +206,10 @@ export default async (job: Job) => {
     });
   } catch (e) {
     await fsPromises.rm(tempFolder, { recursive: true, force: true });
-    logger.error("Error creating audio folder", e);
+    logger.error(`Error creating audio folder for ${trackGroup.id}`);
+    if (e instanceof Error) {
+      logger.error(`${e.message}: ${e.stack}`);
+    }
     return { error: e };
   }
 };
diff --git a/src/queues/moving-files-to-backblaze.ts b/src/queues/moving-files-to-backblaze.ts
@@ -1,8 +1,12 @@
 import { Job, Queue, QueueEvents } from "bullmq";
 import { REDIS_CONFIG } from "../config/redis";
 import logger from "../logger";
-import { getObjectList, minioClient, uploadWrapper } from "../utils/minio";
-import { sleep } from "../jobs/optimize-image";
+import {
+  fileExistCheckBackblaze,
+  getObjectList,
+  minioClient,
+  uploadWrapper,
+} from "../utils/minio";
 import { sendErrorEmail } from "../jobs/send-mail";
 
 const queueOptions = {
@@ -68,10 +72,17 @@ export const moveFilesToBackblazeJob = async (job: Job) => {
   const bucketName = job.data.bucketName;
   const fileName = job.data.fileName;
 
-  logger.info(`moving file: bucket: ${bucketName}, ${fileName}`);
+  logger.info(`checking file exists: ${bucketName}/${fileName}`);
   try {
-    const stream = await minioClient.getObject(bucketName, fileName);
-    await uploadWrapper(bucketName, fileName, stream);
+    const stat = await fileExistCheckBackblaze(bucketName, fileName);
+
+    if (!stat) {
+      logger.info(`file does not exist, moving it: ${bucketName}/${fileName}`);
+      const stream = await minioClient.getObject(bucketName, fileName);
+      await uploadWrapper(bucketName, fileName, stream);
+    } else {
+      logger.info(`file already exists: ${bucketName}/${fileName}`);
+    }
   } catch (e) {
     console.error(e);
     if (e instanceof Error) {

diff --git a/src/static.ts b/src/static.ts
@@ -1,6 +1,7 @@
 import { NextFunction, Request, Response } from "express";
 import {
   backblazeClient,
+  fileExistCheckBackblaze,
   getBufferFromBackblaze,
   getBufferFromMinio,
 } from "./utils/minio";
@@ -13,25 +14,18 @@ export const serveStatic = async (
   res: Response,
   next: NextFunction
 ) => {
-  let backblazeStat;
   let minioStat;
 
-  // FIXME: someone who's better at devops than me will have to figure out
-  // how we can serve these directly from minio
-  try {
-    backblazeStat = await backblazeClient.send(
-      new HeadObjectCommand({
-        Bucket: req.params.bucket,
-        Key: req.params.filename,
-      }),
-      {}
-    );
-    // If the object doesn't exist on backblaze we check in minio
-  } catch (error) {
-    // console.error("failed fetching from backblaze");
-  }
+  const backblazeStat = await fileExistCheckBackblaze(
+    req.params.bucket,
+    req.params.filename
+  );
 
+  // If the object doesn't exist on backblaze we check in minio
   if (!backblazeStat) {
+    logger.error(
+      `Error fetching static file from backblaze, checking minio: ${req.params.bucket}/${req.params.filename}`
+    );
     minioStat = await minioClient.statObject(
       req.params.bucket,
       req.params.filename

diff --git a/src/utils/minio.ts b/src/utils/minio.ts
@@ -8,6 +8,8 @@ import {
   GetObjectCommand,
   DeleteObjectCommand,
   PutObjectCommand,
+  HeadObjectCommand,
+  HeadObjectCommandOutput,
 } from "@aws-sdk/client-s3";
 import fs from "fs";
 import logger from "../logger";
@@ -85,6 +87,25 @@ const createB2BucketIfNotExists = async (bucket: string) => {
   return true;
 };
 
+export const fileExistCheckBackblaze = async (
+  bucket: string,
+  filename: string
+): Promise<HeadObjectCommandOutput | undefined> => {
+  let backblazeStat: HeadObjectCommandOutput | undefined = undefined;
+  try {
+    backblazeStat = await backblazeClient.send(
+      new HeadObjectCommand({
+        Bucket: bucket,
+        Key: filename,
+      }),
+      {}
+    );
+  } catch (e) {
+    return undefined;
+  }
+  return backblazeStat;
+};
+
 export const createBucketIfNotExists = async (
   minioClient: Client,
   bucket: string,