From dcce260b67098ada1eee44a2128a904aed2a197e Mon Sep 17 00:00:00 2001
From: AlexZorzi <Gattorosso123456789>
Date: Fri, 26 Mar 2021 18:36:18 +0100
Subject: [PATCH] push files

---
 MangaWorldApi.md | 129 +++++++++++++++++++++++++++++++++++++
 README.md        |  29 +++++++++
 chapter.go       |  10 +++
 commonLogic.go   |  32 +++++++++
 cred.go          |   6 ++
 download.go      | 164 +++++++++++++++++++++++++++++++++++++++++++++++
 main.go          | 141 ++++++++++++++++++++++++++++++++++++++++
 manga.go         | 115 +++++++++++++++++++++++++++++++++
 pages.go         |   9 +++
 volume.go        |   9 +++
 10 files changed, 644 insertions(+)
 create mode 100644 MangaWorldApi.md
 create mode 100644 README.md
 create mode 100644 chapter.go
 create mode 100644 commonLogic.go
 create mode 100644 cred.go
 create mode 100644 download.go
 create mode 100644 main.go
 create mode 100644 manga.go
 create mode 100644 pages.go
 create mode 100644 volume.go

diff --git a/MangaWorldApi.md b/MangaWorldApi.md
new file mode 100644
index 0000000..4e8de30
--- /dev/null
+++ b/MangaWorldApi.md
@@ -0,0 +1,129 @@
+# Pagina Archivio
+Json Sono caricati in pagina tra '.concat(' & ') \<script>'
+https://www.mangaworld.cc/archive?page=1
+
+Questa è la struttura importante
+ ``` 
+ "o":{
+     "w":[
+         2[
+             "s0-13",
+             2,
+             {
+               "results":2125, //totale manga
+               "mangas":[
+                   {
+                       "extraTitles":[
+                           "Onii-chan is done for",
+                           "お兄ちゃんはおしまい！",
+                           "Onii-chan wa Oshimai"
+                       ]
+                       "author":[
+                        "Nekotoufu"
+                       ],
+                       "artist":[
+                        "Nekotoufu"
+                       ],
+                       "genres":[
+                           {
+                               "name":"commedia",
+                           }
+                           ...
+                       ]
+                       "references":[
+                            "mangaupdatesId":146713,
+                            "mangadexId":22615,
+                            "anilistId":100080,
+                            "malId":112592,
+                       ],
+                       "fansub":{
+                        "name":"Betta Scans",
+                        "link":"https://www.facebook.com/BettaScans/"
+                        },
+                       "vm18":false,
+                       "animeLink":null,
+                       "_id":"5faa2373de75a27e9df97f61",
+                       "title":" Onii-chan Is Done For!",
+                       "status":"ONGOING",
+                       "type":"DOUJINSHI",
+                       "trama":"**", //la trama è qui
+                       "year":2017,
+                       "volumesCount":null,
+                       "chaptersCount":null,
+                       "image":"/mangas/5faa2373de75a27e9df97f61.png",
+                       "slug":"onii-chan-is-done-for",
+                       "slugFolder":"onii-chan-is-done-for",
+                       "linkId":2070,  
+                   },
+                   ... //ripetuto per ogni manga in pagina
+               ],
+             },
+         ],
+     ]
+ }
+ ```
+
+# Pagina Manga
+si trova sempre nello stesso punto nel file html del precedente
+https://www.mangaworld.cc/manga/2070/onii-chan-is-done-for
+la pagina dei manga può essere vista anche così
+https://www.mangaworld.cc/manga/`numero id` quindi può essere ottenuto ciclando da 170 a 2332 (ultimo attuale). così da saltare le query all archivio
+
+```
+"o":
+    "w":[
+         [
+            "s0-2",
+            0,
+            "manga":{
+                // Uguale a "manga" del json precedente
+            },
+         ]
+         3[
+            "s0-83",
+            3,
+            {
+                "pages":{
+                    "volumes":[ // se è manga normale
+                        {
+                            "volume":{
+                                "_id":"5fb01fa0c34214221761baef",
+                                "manga":"5faa2373de75a27e9df97f61",
+                                "name":"Volume 04",
+                                "image":"/volumes/5fb01fa0c34214221761baef.png",
+                            },
+                            "chapters":[
+                                {
+                                "_id":"5fd1f736ea2ffd76512a3647",
+                                "pages":[
+                                        "1.png",
+                                        "2.png",
+                                        "3.png",
+                                        "4.png"
+                                    ],
+                                "name":"Capitolo 33.5",
+
+
+                                },
+                                ...
+                            ],
+                        },
+                        ...
+                    ],
+                    "singleChapters":[ // se è oneshot
+                        {
+                        "_id":"5fc7aedae951863e97b3f121",
+                        "pages":[
+                           "1.jpg",
+                           "2.jpg",
+                            ],
+                        "name":"Oneshot",
+
+                        },
+                        ...
+                    ],
+                },
+            },
+         ],
+
+```
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..3c62074
--- /dev/null
+++ b/README.md
@@ -0,0 +1,29 @@
+# MangaWorld Downloader
+Un Downloader Per mangaworld.cc,
+progetto a scopo didattico non affiliato a mangaworld.cc, non sono responsabile dell'utilizzo e delle immagini hostate su mangaworld.cc
+
+# Features
+- Download di manga interi o singoli capitoli
+- MultiThreaded Download per scaricare le pagine alla massima velocità
+- Download della Preview
+
+# In caso di un CloudFlare captcha
+Per passare il blocco il programma avrà bisogno del cookie cf_clearance e dell'useragent del browser che avete utilzzato per risolvere il captcha
+
+- Token: Risolvete il captcha su www.mangaworld.cc e in Dev Console prendete il valore del cookie cf_clearance 
+  ![Token](https://i.imgur.com/HYUu0M0.png)
+  
+- UserAgent: Al sito www.whatsmyua.info è possibile vedere il proprio useragent
+  ![Useragent](https://i.imgur.com/nZZfCt1.png)
+  
+il programma vi chiederà in automatico questi due parametri se ne avrà bisogno, poi li salverà in automatico in un file JSON così da poterli utilizzare 
+anche dopo che il programma è stato chiuso.
+
+Manualmente: è possibile inserire manualmente i valori in un file chiamato `cred.json` con questa struttura nella stessa cartella dell'eseguibile
+
+```
+{
+"useragent" : "useragent value here"
+"cf_clearance" : "cf_clearance value here"
+}
+```
\ No newline at end of file
diff --git a/chapter.go b/chapter.go
new file mode 100644
index 0000000..a20cba0
--- /dev/null
+++ b/chapter.go
@@ -0,0 +1,10 @@
+package main
+
+type Chapter struct {
+	ID    string   `json:"_id"`
+	Pages []string `json:"pages"`
+	Manga string   `json:"manga"`
+	Name  string   `json:"name"`
+	Slug  string   `json:"slugFolder"`
+}
+
diff --git a/commonLogic.go b/commonLogic.go
new file mode 100644
index 0000000..7fd3710
--- /dev/null
+++ b/commonLogic.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+	"strings"
+)
+
+func Contains(s []int, ID int) bool {
+	for _, v := range s {
+		if v == ID {
+			return true
+		}
+	}
+
+	return false
+}
+
+func Between(value string, a string, b string) string {
+	// Get substring between two strings.
+	posFirst := strings.Index(value, a)
+	if posFirst == -1 {
+		return ""
+	}
+	posLast := strings.Index(value, b)
+	if posLast == -1 {
+		return ""
+	}
+	posFirstAdjusted := posFirst + len(a)
+	if posFirstAdjusted >= posLast {
+		return ""
+	}
+	return value[posFirstAdjusted:posLast]
+}
diff --git a/cred.go b/cred.go
new file mode 100644
index 0000000..822880a
--- /dev/null
+++ b/cred.go
@@ -0,0 +1,6 @@
+package main
+
+type Cred struct {
+	Useragent    string `json:"useragent"`
+	Clearance string `json:"cf_clearance"`
+}
diff --git a/download.go b/download.go
new file mode 100644
index 0000000..8385b50
--- /dev/null
+++ b/download.go
@@ -0,0 +1,164 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+func DownloadManga(m Manga) {
+	// https://cdn.mangaworld.cc/chapters/blust-5fde5e956fe15b440d71d1c6/oneshot-5fde5ea093f7544385e33e4d/1.png
+	// 									 slugManga - idManga / slugChapther - idChapter / pageFilename
+	DownloadPreview(m)
+	// TODO: download Banner
+	// Download Volumes
+	for _, volumeEntity := range m.Pages.Volumes {
+		for _, chapter := range volumeEntity.Chapters {
+			DownloadChapter(chapter, volumeEntity.Volume, m)
+		}
+	}
+	// insert all SingleChapters
+	for _, chapter := range m.Pages.SingleChapters {
+		DownloadChapterNoVolume(chapter, m)
+	}
+}
+
+func DownloadPreview(m Manga) {
+	PATH := filepath.FromSlash("./" + m.Slug + "/")
+	WG.Add(1)
+	go downloadWPath("https://cdn.mangaworld.cc"+m.Image, PATH, m.Slug+"."+strings.Split(m.Image, ".")[1])
+}
+func DownloadChapterNoVolume(c Chapter, m Manga) {
+	WG.Wait()
+	var floatC float64
+	fmt.Println(c.Name)
+	if c.Name == "Oneshot" {
+		floatC = 0.0
+	} else {
+		floattry, err2 := strconv.ParseFloat(strings.Split(c.Name, " ")[1], 32)
+		if err2 != nil {
+			panic(err2)
+		}
+		floatC = floattry
+	}
+	chapterNumber := fmt.Sprintf("%.1F", floatC)
+	for _, page := range c.Pages {
+		API := "https://cdn.mangaworld.cc/chapters/" + m.SlugFolder + "-" + m.ID + "/" + c.Slug + "-" + c.ID + "/" + page
+		PATH := filepath.FromSlash("./" + m.Slug + "/chapter " + chapterNumber + "/")
+		WG.Add(1)
+		go downloadWPath(API, PATH, page)
+	}
+}
+func DownloadChapter(c Chapter, v Volume, m Manga) {
+	floatV, err := strconv.ParseFloat(strings.Split(v.Name, " ")[1], 32)
+	if err != nil {
+		panic(err)
+	}
+	volumeNumber := fmt.Sprintf("%.1f", floatV)
+	fmt.Println("Download: Volume " + v.Name + " Chapter " + c.Name)
+	floatC, err2 := strconv.ParseFloat(strings.Split(c.Name, " ")[1], 32)
+	if err2 != nil {
+		panic(err2)
+	}
+	chapterNumber := fmt.Sprintf("%.1F", floatC)
+	WG.Wait()
+	for _, page := range c.Pages {
+		API := "https://cdn.mangaworld.cc/chapters/" + m.SlugFolder + "-" + m.ID + "/" + v.Slug + "-" + v.ID + "/" + c.Slug + "-" + c.ID + "/" + page
+		PATH := filepath.FromSlash("./" + m.Slug + "/volume " + volumeNumber + "/chapter " + chapterNumber + "/")
+		WG.Add(1)
+		go downloadWPath(API, PATH, page)
+	}
+}
+
+func downloadWPath(url string, path string, filename string) {
+	req, err := http.NewRequest("GET", url, nil)
+	req.Header.Set("content-type", "charset=UTF-8")
+	req.Header.Set("cookie", COOKIE)
+	req.Header.Set("user-agent", USERAGENT)
+	client := &http.Client{}
+	response, e := client.Do(req)
+	if e != nil {
+		log.Fatal(e)
+	}
+	defer response.Body.Close()
+	os.MkdirAll(path, os.ModePerm)
+	file, err := os.Create(path + filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+	_, err = io.Copy(file, response.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	WG.Done()
+}
+
+func GetWebPage(url string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		panic(err)
+	}
+	req.Header.Set("content-type", "charset=UTF-8")
+	req.Header.Set("cookie", COOKIE)
+	req.Header.Set("user-agent", USERAGENT)
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		println(resp.Status)
+		panic(err)
+	}
+	if DEBUG {
+
+		println(resp.Status)
+	}
+	return resp, err
+}
+
+func SyncProxyList() {
+	url := "https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt"
+	resp, err := GetWebPage(url)
+	if err != nil {
+		panic(err)
+	}
+	bodyBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		panic(err)
+	}
+	bodyString := strings.Split(string(bodyBytes), "\n")
+	bodyString = bodyString[2 : len(bodyString)-1]
+	PROXY = bodyString
+}
+func ChangeProxy() {
+	os.Setenv("HTTP_PROXY", PROXY[len(PROXY)-1])
+	os.Setenv("HTTPS_PROXY", PROXY[len(PROXY)-1])
+	println("Proxy changed in " + PROXY[len(PROXY)-1])
+	PROXY = PROXY[0 : len(PROXY)-2]
+}
+
+func ChangeCookie() {
+	reader := bufio.NewReader(os.Stdin)
+	print("Cookie Error, supply new cookie: ")
+	cookie, _ := reader.ReadString('\n')
+	cookienew := strings.ReplaceAll(cookie, "\n", "")
+	COOKIE = "cf_clearance=" + cookienew
+}
+
+func ChangeUserAgent() {
+	reader := bufio.NewReader(os.Stdin)
+	print("supply new USERAGENT: ")
+	useragent, _ := reader.ReadString('\n')
+	useragentnew := strings.ReplaceAll(useragent, "\n", "")
+	USERAGENT = useragentnew
+}
+
+func SaveCookies()  {
+	ioutil.WriteFile("cred.json", []byte("{\n \"useragent\":\""+USERAGENT+"\", \n \"cf_clearance\":\""+COOKIE+"\" \n}"), 0755)
+}
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..f8d6dc9
--- /dev/null
+++ b/main.go
@@ -0,0 +1,141 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+var WG sync.WaitGroup
+var method int = 1
+var DEBUG bool = true
+var COOKIE string = ""
+var USERAGENT string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36"
+var PROXY []string
+var LATESTMANGAIdLink int64
+
+// COOKIE needs only cf_clearance
+/*
+	method:
+			0 database Full sync
+			1 single sync
+			2 telegram event listener
+
+*/
+func main() {
+	//SyncProxyList()
+	//ChangeProxy()
+	getCred()
+	LATESTMANGAIdLink = GetLatestMangaIndex()
+	fmt.Println("##################################################")
+	fmt.Println("#            MangaWorld Downloader               #")
+	fmt.Println("#                                                #")
+	fmt.Println("#            Version: 1.0                        #")
+	fmt.Println("#            Latest Update: 26/03/2021           #")
+	fmt.Println("#                                                #")
+	fmt.Println("##################################################")
+	single()
+	WG.Wait()
+}
+
+func getCred(){
+
+	code, err := GetWebPage("https://www.mangaworld.cc/")
+	if(err == nil){
+		if(code.StatusCode == 200){
+			return
+		}
+	}
+	if _, err := os.Stat("./cred.json"); !os.IsNotExist(err) {
+		data, _ := ioutil.ReadFile("./cred.json")
+		var cred Cred
+		json.Unmarshal(data, &cred)
+		COOKIE = cred.Clearance
+		USERAGENT = cred.Useragent
+	}else {
+		fmt.Println("The file cred.json dosen't exist!, creating one...")
+		ChangeCookie()
+		ChangeUserAgent()
+		SaveCookies()
+	}
+}
+
+func single() {
+	reader := bufio.NewReader(os.Stdin)
+	fmt.Print("Enter Manga Url/ID to Check: ")
+	Url, _ := reader.ReadString('\n')
+	UrlClean := strings.ReplaceAll(Url, "\n", "")
+	// sample url https://www.mangaworld.cc/manga/1807/murim-login/
+	var id int64
+	for _, idFind := range strings.Split(UrlClean, "/") {
+		idInt, err := strconv.ParseInt(idFind, 10, 64)
+		if err == nil {
+			id = idInt
+			break
+		}
+	}
+	if id == 0 && id >= 170 && id <= LATESTMANGAIdLink { // check if an id is found
+		fmt.Println("Error: Url Not Valid, Quitting...")
+		os.Exit(0)
+	}
+	fmt.Println("Info: Anime id = " + strconv.Itoa(int(id)))
+	mangasaved, err := GetManga(int(id))
+	if err != nil {
+		println(err.Error())
+	}
+	fmt.Println("Selected: "+mangasaved.Title)
+	fmt.Print("0 for single 1 for whole: ")
+	whole, _ := reader.ReadString('\n')
+	wholecl := strings.ReplaceAll(whole, "\n", "")
+	if wholecl == "0" {
+		if len(mangasaved.Pages.Volumes) > 0 {
+			fmt.Println("Pick: What Volume You wanna add/update? ")
+			var aviableVolumes []struct {
+				Volume   Volume    `json:"volume"`
+				Chapters []Chapter `json:"chapters"`
+			}
+			for index, volume := range mangasaved.Pages.Volumes {
+				println(index, ")    PickOption: Volume "+volume.Volume.Name)
+				aviableVolumes = append(aviableVolumes, volume)
+			}
+			pick, _ := reader.ReadString('\n')
+			pickClean, _ := strconv.Atoi(strings.ReplaceAll(pick, "\n", ""))
+			selectedVolume := aviableVolumes[pickClean]
+			fmt.Println("Pick: What Chapter You wanna add/update? ")
+			var aviableChapters []Chapter
+			for index, chapter := range selectedVolume.Chapters {
+				println(index, ")  PickOption: Chapters "+chapter.Name)
+				aviableChapters = append(aviableChapters, chapter)
+			}
+			pick, _ = reader.ReadString('\n')
+			pickClean, _ = strconv.Atoi(strings.ReplaceAll(pick, "\n", ""))
+			selectedChapter := aviableChapters[pickClean]
+			fmt.Println("Starting Chapter Download")
+			DownloadChapter(selectedChapter, selectedVolume.Volume, mangasaved)
+			DownloadPreview(mangasaved)
+		} else {
+			fmt.Println("Pick: What Chapter You wanna add/update? ")
+			var aviableChapters []Chapter
+			for index, chapter := range mangasaved.Pages.SingleChapters {
+				println(index, ")  PickOption: Chapters "+chapter.Name)
+				aviableChapters = append(aviableChapters, chapter)
+			}
+			pick, _ := reader.ReadString('\n')
+			pickClean, _ := strconv.Atoi(strings.ReplaceAll(pick, "\n", ""))
+			selectedChapter := aviableChapters[pickClean]
+			fmt.Println("Starting Chapter Download")
+			DownloadChapterNoVolume(selectedChapter, mangasaved)
+			DownloadPreview(mangasaved)
+
+		}
+	} else if wholecl == "1" {
+		DownloadManga(mangasaved)
+	} else {
+		println("Error")
+	}
+}
\ No newline at end of file
diff --git a/manga.go b/manga.go
new file mode 100644
index 0000000..4d6b222
--- /dev/null
+++ b/manga.go
@@ -0,0 +1,115 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"github.com/buger/jsonparser"
+	"net/http"
+	"strconv"
+)
+
+type Manga struct {
+	ID          string   `json:"_id"`
+	IDLink      int      `json:"linkId"`
+	Title       string   `json:"title"`
+	ExtraTitles []string `json:"extraTitles"`
+	Author      []string `json:"author"`
+	Artist      []string `json:"artist"`
+	Genres      []struct {
+		Slug string `json:"slug"`
+	}
+	References struct {
+		MangaupdatesId int `json:"mangaupdatesId"`
+		MangadexId     int `json:"mangadexId"`
+		AnilistId      int `json:"anilistId"`
+		MalId          int `json:"malId"`
+	}
+	Fansub struct {
+		Name string `json:"name"`
+		Link string `json:"Link"`
+	}
+	Vm18          bool   `json:"vm18"`
+	Status        string `json:"status"`
+	Type          string `json:"type"`
+	Trama         string `json:"trama"`
+	Year          int    `json:"year"`
+	ChaptersCount string `json:"chaptersCount"`
+	VolumesCount  string `json:"volumesCount"`
+	Image         string `json:"image"`
+	Slug          string `json:"slug"`
+	SlugFolder    string `json:"slugFolder"`
+	ReadMode      string `json:"readMode"`
+	Pages         Pages
+}
+
+//o.w[1][2].globalData.latestMangas
+func bufToJson(response *http.Response) []byte {
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(response.Body)
+	BodyStr := buf.String()
+	jsondata := Between(BodyStr, ".concat(", ")</script>")
+	data := []byte(jsondata)
+	return data
+}
+
+func GetLatestMangaIndex() int64 {
+	url := "https://www.mangaworld.cc/"
+	var latest int64
+	for {
+		resp, err := GetWebPage(url)
+		data := bufToJson(resp)
+		latestMangaTemp, err := jsonparser.GetInt(data, "o", "w", "[1]", "[2]", "globalData", "latestMangas", "[0]", "linkId")
+		if err != nil {
+			//println("CloudFlare Error refreshing proxy")
+			//ChangeProxy()
+			println("CloudFlare Error refreshing cookie")
+			ChangeCookie()
+			ChangeUserAgent()
+			SaveCookies()
+		} else {
+			latest = latestMangaTemp
+			break
+		}
+	}
+	return latest
+}
+
+func GetManga(ID int) (Manga, error) {
+	url := "https://www.mangaworld.cc/manga/" + strconv.Itoa(ID)
+	var mangadata []byte
+	var readdata []byte
+	for {
+		resp, err := GetWebPage(url)
+		if resp.StatusCode == 404 {
+			return Manga{}, errors.New("Error 404 skip of ID")
+		}
+		data := bufToJson(resp)
+		mangadataTemp, _, _, err := jsonparser.Get(data, "o", "w", "[0]", "[2]", "manga")
+		readdataTemp, _, _, err2 := jsonparser.Get(data, "o", "w", "[3]", "[2]", "pages")
+		if err2 != nil {
+			readdataTemp, _, _, err2 = jsonparser.Get(data, "o", "w", "[4]", "[2]", "pages")
+		}
+		if err != nil && err2 != nil {
+			//println("CloudFlare Error refreshing proxy")
+			//ChangeProxy()
+			println(url)
+			println("CloudFlare Error refreshing cookie")
+			ChangeCookie()
+			ChangeUserAgent()
+			SaveCookies()
+		} else {
+			mangadata = mangadataTemp
+			readdata = readdataTemp
+			break
+		}
+	}
+	var manga Manga
+	json.Unmarshal(mangadata, &manga)
+
+	var pages Pages
+	json.Unmarshal(readdata, &pages)
+
+	manga.Pages = pages
+	return manga, nil
+}
diff --git a/pages.go b/pages.go
new file mode 100644
index 0000000..3dbe6fd
--- /dev/null
+++ b/pages.go
@@ -0,0 +1,9 @@
+package main
+
+type Pages struct {
+	Volumes []struct {
+		Volume   Volume    `json:"volume"`
+		Chapters []Chapter `json:"chapters"`
+	} `json:"volumes"`
+	SingleChapters []Chapter `json:"singleChapters"`
+}
diff --git a/volume.go b/volume.go
new file mode 100644
index 0000000..9887b74
--- /dev/null
+++ b/volume.go
@@ -0,0 +1,9 @@
+package main
+
+type Volume struct {
+	ID    string `json:"_id"`
+	Manga string `json:"manga"`
+	Name  string `json:"name"`
+	Slug  string `json:"slugFolder"`
+	Image string `json:"image"`
+}