chore: documentation, move some logging to debug level, minor cleanup

Signed-off-by: Fredrik Lundkvist <[email protected]>

readme.md

@@ -1,19 +1,68 @@
-# {{Name}}
+# Ad Normalizer
 
-<!--
+A Proxy put in fron of an ad server that dispatches transcoding and packaging of VAST creatives.
+
+The service accepts requests to the endpoint `api/v1/vast`, and returns a JSON array with the following structure:
+
+```json
+{
+"assets": [
+    {
+        "creativeId": "abcd1234",
+        "masterPlaylistUrl": "https://your-minio-endpoint/creativeId/substring/index.m3u8
+    },
+]
+}
+```
+
+The service uses redis to keep track of transcoded creatives, and returns the master playlist URL if one is found; if the service does not know of any packaged assets for a creative, it creates a transcoding and packaging pipeline, and monitors the provided minio bucket for asset uploads. Once the assets are in place, the master playlist URL is added to the redis cache.
 
 ## Requirements
-Add any external project dependencies such as node.js version etc here
 
-## Installation / Usage
+To run the ad normalizer as a service, the following other services are needed
+
+- A redis instance
+
+A media processing pipeline consisting of the following:
+
+- A running instance of [SVT Encore](https://github.com/svt/encore)
+- A service that handles encore callbacks
+- A packaging service to handle the transcoded files
+- A minio bucket for the packaged assets
+
+Such a pipeline can easily be created using [Eyevinn open source cloud](https://docs.osaas.io/osaas.wiki/Solution%3A-VOD-Transcoding.html)
+
+Note: the ad normalizer assumes that your packager is set up with the output subfolder template `$EXTERNALID$/$JOBID$`
+
+## Usage
+
+### Environment variables
+
+| Variable                | Description                                                                                                                | Default value |
+| ----------------------- | -------------------------------------------------------------------------------------------------------------------------- | ------------- |
+| `ENCORE_URL`            | The URL of your encore instance                                                                                            | none          |
+| `CALLBACK_LISTENER_URL` | The URL of your callback listener                                                                                          | none          |
+| `MINIO_URL`             | The minio instance endpoint                                                                                                | none          |
+| `MINIO_ACCESS_KEY`      | Your minio access key                                                                                                      | none          |
+| `MINIO_SECRET_KEY`      | Your minio secret key                                                                                                      | none          |
+| `LOG_LEVEL`             | The log level of the service                                                                                               | Info          |
+| `REDIS_URL`             | The url of your redis instance                                                                                             | none          |
+| `AD_SERVER_URL`         | The url of your ad server                                                                                                  | none          |
+| `PORT`                  | The port that the server listens on                                                                                        | 8000          |
+| `MINIO_BUCKET`          | The bucket that the packaged assets will be placed into                                                                    | none          |
+| `SERVICE_ACCESS_TOKEN`  | your OSC service access token for encore (only needed when running the service outside OSC with an encore instance in OSC) | none          |
+
+### starting the service
 
-Add clear instructions on how to use the project here
+`npm run start`
 
 ## Development
 
-Add clear instructions on how to start development of the project here
+When developing, it is highly recommended that you put the required variables in a dotenv file at the repository root. This will make it easier to iterate and change environment variables throughout the development process.
+Before pushing changes to the repo, please run the following steps to make sure your pipeline will succeed:
 
--->
+- `npm run test` to verify that your changes do not break existing functionality (if adding features, it is good practice to also write tests).
+- `npm run lint` as well as `npm run pretty` to ensure that the code still follows the formatting standards. Errors should be fixed, as the pipeline won't succeed otherwise. Warnings should be handled on a case-by-case basis.
 
 ### Contributing
 

src/api.ts

@@ -68,7 +68,7 @@ export default (opts: ApiOptions) => {
     logger.info('Saving to Redis', { key, value });
     redisclient.set(key, value);
   };
-  logger.info(config.callbackListenerUrl);
+  logger.debug('callback listener URL:', config.callbackListenerUrl);
   const encoreClient = new EncoreClient(
     config.encoreUrl,
     config.callbackListenerUrl,
@@ -110,11 +110,11 @@ export default (opts: ApiOptions) => {
   api.register(vastApi, {
     adServerUrl: config.adServerUrl,
     assetServerUrl: `https://${config.minioUrl}/${config.minioBucket}/`,
-    lookUpAsset: async (mediaFile: string) =>  redisclient.get(mediaFile),
+    lookUpAsset: async (mediaFile: string) => redisclient.get(mediaFile),
     onMissingAsset: async (asset: ManifestAsset) =>
       encoreClient.createEncoreJob(asset),
     setupNotification: (asset: ManifestAsset) => {
-      logger.info('Setting up notification for asset', { asset });
+      logger.debug('Setting up notification for asset', { asset });
       minioClient.listenForNotifications(
         config.minioBucket,
         asset.creativeId + '/', // TODO: Pass encore job id and add as part of the prefix

src/encore/encoreclient.ts

@@ -21,7 +21,6 @@ export class EncoreClient {
         ...headers,
         'x-jwt': `Bearer ${this.serviceAccessToken}`
       };
-      const sat = this.serviceAccessToken;
     }
     return fetch(`${this.url}/encoreJobs`, {
       method: 'POST',

src/minio/minio.ts

@@ -21,7 +21,7 @@ export class MinioClient {
   ) {}
 
   setupClient() {
-    logger.info('Setting up Minio Client', { url: this.url });
+    logger.debug('Setting up Minio Client', { url: this.url });
     if (this.minioclient) {
       logger.error('Minio client already connected');
       return;
@@ -39,7 +39,7 @@ export class MinioClient {
     masterPlaylistName: string,
     onNotification: (r: any) => Promise<void>
   ) => {
-    logger.info('Listening for notifications', {
+    logger.debug('Listening for notifications', {
       bucketName,
       assetId,
       masterPlaylistName
@@ -58,10 +58,10 @@ export class MinioClient {
     if (poller == undefined) {
       logger.error('Failed to create poller');
     } else {
-      logger.info('Poller created');
+      logger.debug('Poller created');
     }
     poller?.on('notification', (record) => {
-      logger.info('Received notification', record);
+      logger.debug('Received notification', record);
       onNotification(record as MinioNotification);
       poller.stop();
     });

src/redis/redisclient.ts

@@ -12,7 +12,7 @@ export class RedisClient {
       return;
     }
     this.client = await createClient({ url: this.url })
-      .on('error', (err) => logger.error('Redis error',  err ))
+      .on('error', (err) => logger.error('Redis error', err))
       .connect();
   }
 

src/server.ts

@@ -1,6 +1,5 @@
 import api from './api';
 import 'dotenv/config';
-import getConfiguration from './config/config';
 import logger from './util/logger';
 
 const server = api({ title: '@eyevinn/typescript-nodejs' });
@@ -11,7 +10,7 @@ server.listen({ port: PORT, host: '0.0.0.0' }, (err, address) => {
   if (err) {
     throw err;
   }
-  console.log(`Server listening on ${address}`);
+  logger.info(`Server listening on ${address}`);
 });
 
 export default server;

src/vast/vastApi.ts

@@ -46,9 +46,9 @@ export const vastApi: FastifyPluginCallback<AdApiOptions> = (
         creatives,
         opts.lookUpAsset
       );
-      logger.info('Partitioned creatives', { found, missing });
-      logger.info('Received creatives', { creatives });
-      logger.info('Received VAST request');
+      logger.debug('Partitioned creatives', { found, missing });
+      logger.debug('Received creatives', { creatives });
+      logger.debug('Received VAST request');
       missing.forEach(async (creative) => {
         if (opts.onMissingAsset) {
           opts
@@ -65,16 +65,15 @@ export const vastApi: FastifyPluginCallback<AdApiOptions> = (
                 });
                 throw new Error('Failed to submit encore job');
               }
-              logger.info('Submitted encore job', { creative });
               return response.json();
             })
             .then((data) => {
               const encoreJobId = data.id;
-              logger.info('Submitted encore job', { encoreJobId });
+              logger.info('Submitted encore job', { encoreJobId, creative });
               if (opts.setupNotification) {
-                logger.info('Setting up notification');
+                logger.debug('Setting up notification');
                 opts.setupNotification(creative);
-                logger.info("Notification set up. You're good to go!");
+                logger.debug("Notification set up. You're good to go!");
               }
             })
             .catch((error) => {
@@ -101,7 +100,7 @@ const partitionCreatives = async (
   const [found, missing]: [ManifestAsset[], ManifestAsset[]] = [[], []];
   for (const creative of creatives) {
     const asset = await lookUpAsset(creative.creativeId);
-    logger.info('Looking up asset', { creative, asset });
+    logger.debug('Looking up asset', { creative, asset });
     if (asset) {
       found.push({ creativeId: creative.creativeId, masterPlaylistUrl: asset });
     } else {
@@ -130,13 +129,13 @@ const getCreatives = async (
         throw new Error('Response from ad server was not OK');
       }
       const contentType = response.headers.get('content-type');
-      logger.info('Received response from ad server', { contentType });
+      logger.debug('Received response from ad server', { contentType });
       return response.text();
     })
     .then((body) => {
       const parser = new XMLParser();
       const parsedVAST = parser.parse(body);
-      const creatives = parsedVAST.VAST.Ad.reduce((acc: any[], ad: any) => {
+      const creatives = parsedVAST.VAST.Ad.reduce((acc: ManifestAsset[], ad: any) => {
         const adId = ad.InLine.Creatives.Creative.UniversalAdId.replace(
           /[^a-zA-Z0-9]/g,
           ''