Merge pull request #12 from pararang/feat/caching-with-goroutines

feat: caching scraping result with goroutines

service/scraper/scraper.go

@@ -118,10 +118,12 @@ func (s *scraper) GetProvinceAvailability(provinceID int) ([]model.HospitalSumma
 		data = append(data, *hospital)
 	})
 
-	err = redis.SetScrapedAvailableHospitals(url, data)
-	if err != nil {
-		log.Println("ERROR: Error set to redis")
-	}
+	go func() {
+		err = redis.SetScrapedAvailableHospitals(url, data)
+		if err != nil {
+			log.Println("ERROR: Error set to redis")
+		}
+	}()
 
 	return data, nil
 }
@@ -134,14 +136,23 @@ func (s *scraper) GetHospitalDetail(hospitalCode string) (model.HospitalDetail,
 	}
 
 	var data model.HospitalDetail
+	var provinceID = hospitalCode[0:2]
+	var url = fmt.Sprintf("https://yankes.kemkes.go.id/app/siranap/tempat_tidur?kode_rs=%s&jenis=1&propinsi=%sprop&kabkota=", hospitalCode, provinceID)
 
-	provinceID := hospitalCode[0:2]
-
-	domHTML, err := s.readPage(fmt.Sprintf("https://yankes.kemkes.go.id/app/siranap/tempat_tidur?kode_rs=%s&jenis=1&propinsi=%sprop&kabkota=", hospitalCode, provinceID))
+	domHTML, err := s.readPage(url)
 	if err != nil {
 		return data, err
 	}
 
+	cachedData, _ := redis.GetScrapedDetailHospital(url)
+	if len(cachedData) > 0 {
+		err := json.Unmarshal([]byte(cachedData), &data)
+		if err == nil {
+			log.Printf("INFO: Return data from cached data")
+			return data, err
+		}
+	}
+
 	titleSelector := domHTML.Find("p[class=mb-0]").First()
 
 	data.Address = strings.TrimSpace(titleSelector.Find("small").First().Text())
@@ -189,6 +200,13 @@ func (s *scraper) GetHospitalDetail(hospitalCode string) (model.HospitalDetail,
 
 	data.Room = rooms
 
+	go func() {
+		err = redis.SetScrapedDetailHospital(url, data)
+		if err != nil {
+			log.Println("ERROR: Error set to redis")
+		}
+	}()
+
 	return data, nil
 }
 

storage/redis.go

@@ -14,8 +14,12 @@ import (
 type Redis interface {
 	Get(key string) (string, error)
 	Set(key string, value string) error
+	SetEx(key string, value string) error
+
 	SetScrapedAvailableHospitals(hashURL, hashHTML string, hospitals []model.HospitalSummary) error
 	GetScrapedAvailableHospitals(key string) (string, error)
+	SetScrapedDetailHospital(hashURL, hashHTML string, hospital model.HospitalDetail) error
+	GetScrapedDetailHospital(key string) (string, error)
 }
 
 type redis struct {
@@ -72,3 +76,23 @@ func buildKeyAvailableHospital(url string) string {
 	const prefix = "available_hospitals"
 	return prefix + "." + utils.GetMD5String(url)
 }
+
+// SetScrapedAvailableHospitals sets the scraped available hospitals
+func (r redis) SetScrapedDetailHospital(url string, hospitals model.HospitalDetail) error {
+	var key = buildKeyDetailHospital(url)
+	var value = utils.JSONString(hospitals)
+	var expireTime = time.Duration(5 * 60 * time.Second) // TODO: set to env var
+
+	return r.SetEx(key, value, expireTime)
+}
+
+// GetScrapedAvailableHospitals sets the scraped available hospitals
+func (r redis) GetScrapedDetailHospital(url string) (string, error) {
+	return r.Get(buildKeyDetailHospital(url))
+}
+
+// buildKeyAvailableHospital returns the key for the scraped available hospitals
+func buildKeyDetailHospital(url string) string {
+	const prefix = "detail_hospital"
+	return prefix + "." + utils.GetMD5String(url)
+}