Update website

.github/workflows/deploy.yaml

@@ -19,9 +19,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up pnpm
-        uses: pnpm/action-setup@v3
+        uses: pnpm/action-setup@v4
         with:
-          version: 8
+          version: 9
 
       - name: Set up Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
@@ -33,6 +33,8 @@ jobs:
 
       - name: Generate JSON containing all concerts
         run: pnpm rust
+        env:
+          RUST_LOG: debug
 
       - name: Generate TypeScript types from Rust structs
         run: pnpm rust:types

rust/Cargo.toml

@@ -16,6 +16,8 @@ chrono-tz = "0.9.0"
 deunicode = "1.6.0"
 futures = "0.3.30"
 html-escape = "0.2.13"
+log = "0.4.22"
+pretty_env_logger = "0.5.0"
 regex = "1.10.4"
 reqwest = {version = "0.12.3", features = ["json"]}
 scraper = "0.19.0"

rust/src/core.rs

@@ -1,6 +1,7 @@
 use chrono::{DateTime, Utc};
 use chrono_tz::Europe::London;
 use deunicode::deunicode;
+use log::info;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
 use ts_rs::TS;
@@ -58,5 +59,5 @@ pub fn add_id_to_concert(c: ConcertData) -> Concert {
 
 pub fn report_concert(c: &ConcertData) {
     let london_datetime = c.datetime.with_timezone(&London);
-    eprintln!("Found concert on {}: {}", london_datetime, c.title);
+    info!("Found {}: {}", london_datetime, c.title);
 }

rust/src/main.rs

@@ -1,6 +1,7 @@
 use chrono::{DateTime, TimeZone, Utc};
 use chrono_tz::Europe::London;
 use london_classical::core;
+use log::debug;
 use serde::Serialize;
 use std::fs::{create_dir_all, File};
 
@@ -42,17 +43,27 @@ fn display_programme(concert: &Concert) {
 
 #[tokio::main]
 async fn main() {
-    use london_classical::proms;
+    pretty_env_logger::init();
     let client = reqwest::Client::new();
 
+    // Fetch Wigmore concerts, first reading from $WIGMORE_MAX and defaulting to 220 if not given.
+    // If you push it a bit more it starts to rate limit
     use london_classical::wigmore;
-
-    // Fetch Wigmore concerts. If you push it a bit more it starts to rate limit
-    let mut wigmore_concerts = wigmore::get_concerts(&client, Some(220)).await;
+    let max_wigmore_concerts = {
+        match std::env::var("WIGMORE_MAX") {
+            Ok(s) => match s.as_str() {
+                "all" => None,
+                _ => Some(s.parse::<usize>().unwrap()),
+            },
+            Err(_) => Some(220),
+        }
+    };
+    debug!("max_wigmore_concerts: {:?}", max_wigmore_concerts);
+    let mut wigmore_concerts = wigmore::get_concerts(&client, max_wigmore_concerts).await;
 
     // Fetch Proms
-    let mut proms_concerts =
-        proms::scrape(proms::PROMS_2024_URL, &client).await;
+    use london_classical::proms;
+    let mut proms_concerts = proms::scrape(proms::PROMS_2024_URL, &client).await;
 
     // Concatenate and sort
     let mut full_concerts = vec![];
@@ -67,7 +78,10 @@ async fn main() {
         .collect();
 
     // Check uniqueness of IDs
-    let mut all_ids: Vec<&str> = full_concerts_with_ids.iter().map(|c| c.id.as_str()).collect();
+    let mut all_ids: Vec<&str> = full_concerts_with_ids
+        .iter()
+        .map(|c| c.id.as_str())
+        .collect();
     all_ids.sort();
     for i in 0..all_ids.len() - 1 {
         if all_ids[i] == all_ids[i + 1] {

rust/src/proms.rs

@@ -1,6 +1,7 @@
 use crate::core;
 use chrono::{NaiveDate, TimeZone, Utc};
 use chrono_tz::Europe::London;
+use log::info;
 use regex::Regex;
 use scraper::{ElementRef, Html, Selector};
 use std::cmp::min;
@@ -9,9 +10,7 @@ pub const PROMS_2024_URL: &str = "https://www.bbc.co.uk/events/rfbp5v/series";
 
 // Scrapes concerts from BBC Proms website
 pub async fn scrape(url: &str, client: &reqwest::Client) -> Vec<core::ConcertData> {
-    println!("----------------------------------------");
-    println!("Scraping BBC Proms from URL: {}", url);
-    println!("----------------------------------------");
+    info!("Scraping BBC Proms from URL: {}", url);
 
     let html: String = client
         .get(url)
@@ -38,6 +37,7 @@ pub async fn scrape(url: &str, client: &reqwest::Client) -> Vec<core::ConcertDat
             .for_each(|concert| concerts.push(concert));
     }
 
+    info!("Scraped {} Proms", concerts.len());
     concerts
 }
 

rust/src/wigmore.rs

@@ -3,6 +3,7 @@ use chrono::{DateTime, Utc};
 use futures::future::join_all;
 use futures::stream::{self, StreamExt};
 use html_escape::decode_html_entities;
+use log::{info, warn};
 use regex::Regex;
 use scraper::{Html, Selector};
 use serde::{Deserialize, Serialize};
@@ -13,9 +14,7 @@ pub async fn get_concerts(client: &reqwest::Client, max: Option<usize>) -> Vec<c
         Some(n) => format!("first {}", n),
         None => "all".to_string(),
     };
-    println!("----------------------------------------");
-    println!("Scraping {} Wigmore Hall concerts", max_text);
-    println!("----------------------------------------");
+    info!("Scraping {max_text} Wigmore Hall concerts");
 
     let wigmore_intermediate_concerts = get_api(client).await;
     let slice = match max {
@@ -33,6 +32,7 @@ pub async fn get_concerts(client: &reqwest::Client, max: Option<usize>) -> Vec<c
         .collect::<Vec<core::ConcertData>>();
 
     wigmore_concerts.sort_by_key(|concert| concert.datetime);
+    info!("Scraped {} Wigmore Hall concerts", wigmore_concerts.len());
     wigmore_concerts
 }
 
@@ -135,7 +135,7 @@ async fn get_full_concert(
     match json_result {
         Ok(json) => Some(parse_concert_json(fp_entry, json)),
         Err(e) => {
-            eprintln!("Error parsing JSON for concert at {}: {}", fp_entry.url, e);
+            warn!("Error parsing JSON for concert at {}: {}", fp_entry.url, e);
             None
         }
     }
@@ -149,7 +149,7 @@ fn parse_concert_json(
     let mut pieces = Vec::new();
     let opt_repertoire = json["data"]["page"]["repertoire"].as_array();
     match opt_repertoire {
-        None => eprintln!("No repertoire found for concert at {}", fp_entry.url),
+        None => warn!("No repertoire found for concert at {}", fp_entry.url),
         Some(repertoire) => {
             for piece in repertoire {
                 let opt_cycle = piece["cycle"].as_str().map(decode_html_entities);
@@ -183,7 +183,7 @@ fn parse_concert_json(
     let mut performers = Vec::new();
     let opt_credits = json["data"]["page"]["credits"].as_array();
     match opt_credits {
-        None => eprintln!("No performer credits found for concert at {}", fp_entry.url),
+        None => warn!("No performer credits found for concert at {}", fp_entry.url),
         Some(credits) => {
             for credit in credits {
                 let opt_artist_name = credit["artist"]["title"].as_str().map(decode_html_entities);