replaced colly with surf to restore functionality

main.go

@@ -10,10 +10,11 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/gocolly/colly/v2"
 	"github.com/schollz/progressbar/v3"
+	"gopkg.in/headzoo/surf.v1"
 )
 
 var simplePtr *bool
@@ -35,14 +36,15 @@ type chapter struct {
 
 func extractSeriesInfo(url string) (string, string) {
 	// Pattern: /series/{ID}/{name}
-	c := colly.NewCollector()
+
+	bow := surf.NewBrowser()
+
+	bow.Open(url)
 
 	var name string
-	c.OnHTML("#top", func(e *colly.HTMLElement) {
-		name = e.DOM.Find("h1").First().Text()
-	})
 
-	c.Visit(url)
+	name = bow.Find("h1").First().Text()
+
 	re := regexp.MustCompile(`/series/([^/]+)`)
 	matches := re.FindStringSubmatch(url)
 	if len(matches) < 2 {
@@ -67,113 +69,104 @@ func FormatChapterString(s string) string {
 func getAllChapters(seriesID string, single float64) []chapter {
 	url := fmt.Sprintf("https://weebcentral.com/series/%s/full-chapter-list", seriesID)
 
-	c := colly.NewCollector()
+	bow := surf.NewBrowser()
 
-	p := &pageInfo{Links: make(map[string]int)}
+	err := bow.Open(url)
+	if err != nil {
+		panic(err)
+	}
 
 	chapt := []chapter{}
 
-	// count links
-	c.OnHTML("div", func(e *colly.HTMLElement) {
+	bow.Find("div a").Each(func(i int, h *goquery.Selection) {
+		link, _ := h.Attr("href")
+		name := strings.Split(h.Find("span span").First().Text(), " ")[1]
+		chNum := FormatChapterString(name)
 
-		e.DOM.Find("a").Each(func(i int, h *goquery.Selection) {
-			link, _ := h.Attr("href")
-			name := strings.Split(h.Find("span span").First().Text(), " ")[1]
-			chNum := FormatChapterString(name)
-
-			if single != 9999 {
-				if name == fmt.Sprintf("%g", single) {
-					chapt = append(chapt, chapter{
-						Name: "Chapter " + chNum,
-						Link: link,
-					})
-				}
-
-			} else {
+		if single != 9999 {
+			if name == fmt.Sprintf("%g", single) {
 				chapt = append(chapt, chapter{
 					Name: "Chapter " + chNum,
 					Link: link,
 				})
 			}
-		})
 
+		} else {
+			chapt = append(chapt, chapter{
+				Name: "Chapter " + chNum,
+				Link: link,
+			})
+		}
 	})
 
-	// extract status code
-	c.OnResponse(func(r *colly.Response) {
-		log.Println("response received", r.StatusCode)
-		p.StatusCode = r.StatusCode
-	})
-	c.OnError(func(r *colly.Response, err error) {
-		log.Println("error:", r.StatusCode, err)
-		p.StatusCode = r.StatusCode
-	})
-
-	c.Visit(url)
-
-	//slices.Reverse(chapt)
 	return chapt
 }
 
 var wg sync.WaitGroup
 
 func downloadImage(url string, path string) {
+	bow := surf.NewBrowser()
 
-	c := colly.NewCollector()
+	err := bow.Open(url)
+	if err != nil {
+		log.Fatal(err)
+	}
 
-	c.OnResponse(func(r *colly.Response) {
-		defer wg.Done()
-		fullFileName := strings.Split(r.FileName(), "_")
-		newFileName := fullFileName[len(fullFileName)-1]
-		r.Save(path + newFileName)
-	})
-	c.Visit(url)
+	fullFileName := strings.Split(url, "-")
+	newFileName := fullFileName[len(fullFileName)-1]
+
+	//open a file for writing
+	file, err := os.Create(path + newFileName)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	bow.Download(file)
 }
 
 func downloadChapter(chapterPath string, url string) {
 	os.Mkdir(chapterPath, 0775)
-	c := colly.NewCollector()
+
+	bow := surf.NewBrowser()
+
+	err := bow.Open(url)
+	if err != nil {
+		panic(err)
+	}
+
+	images := bow.Images()
+
+	imagesLength := len(images)
+	//open a file for writing
+
 	count := 0
-	c.OnHTML("section", func(h *colly.HTMLElement) {
-		images := h.DOM.Find("img")
-		imagesLength := images.Length()
 
-		bar := progressbar.NewOptions(imagesLength,
-			progressbar.OptionEnableColorCodes(true),
-			progressbar.OptionFullWidth(),
-			progressbar.OptionShowCount(),
-			progressbar.OptionSetDescription("[cyan][1/2][reset] Downloading"),
-			progressbar.OptionSetTheme(progressbar.Theme{
-				Saucer:        "[green]=[reset]",
-				SaucerHead:    "[green]>[reset]",
-				SaucerPadding: " ",
-				BarStart:      "[",
-				BarEnd:        "]",
-			}))
-		images.Each(func(i int, s *goquery.Selection) {
-			imgSrc, _ := s.Attr("src")
-			if !*simplePtr {
-				bar.Add(1)
-			}
-			wg.Add(1)
-			count++
-			go downloadImage(imgSrc, chapterPath+"/")
-			// with this I can download 10 images at the same time
-			if count == 10 || i == imagesLength-1 {
-				wg.Wait()
-				count = 0
-			}
-		})
-	})
-
-	// extract status code
-	//c.OnResponse(func(r *colly.Response) {
-	//	log.Println("response received", r.StatusCode)
-	//})
-	c.OnError(func(r *colly.Response, err error) {
-		log.Println("error:", r.StatusCode, err)
-	})
-	c.Visit(fmt.Sprint(url, "/images"))
+	bar := progressbar.NewOptions(imagesLength,
+		progressbar.OptionEnableColorCodes(true),
+		progressbar.OptionFullWidth(),
+		progressbar.OptionShowCount(),
+		progressbar.OptionSetDescription("[cyan][1/2][reset] Downloading Chapter"),
+		progressbar.OptionSetTheme(progressbar.Theme{
+			Saucer:        "[green]=[reset]",
+			SaucerHead:    "[green]>[reset]",
+			SaucerPadding: " ",
+			BarStart:      "[",
+			BarEnd:        "]",
+		}))
+	for _, img := range images {
+		imgSrc := img.Url().String()
+		if !*simplePtr {
+			bar.Add(1)
+		}
+		wg.Add(1)
+		count++
+		downloadImage(imgSrc, chapterPath+"/")
+		// with this I can download 10 images at the same time
+		//if count == 10 || i == imagesLength-1 {
+		//	wg.Wait()
+		//	count = 0
+		//}
+	}
 }
 
 func compressToCbz(chapterPath string) {
@@ -247,8 +240,8 @@ func downloadSeries(url string, downloadPath string, single float64) {
 		chapterPath := fmt.Sprintf("%s/%s", seriesPath, c.Name)
 		if _, err := os.Stat(seriesPath + "/" + c.Name + ".cbz"); os.IsNotExist(err) {
 			log.Println("Downloading ", c.Name)
-			downloadChapter(chapterPath, "https://weebcentral.com"+c.Link)
-			//log.Println("Compressing", c.Name)
+			downloadChapter(chapterPath, "https://weebcentral.com"+c.Link+"/images")
+			log.Println("Compressing", c.Name)
 			compressToCbz(chapterPath)
 			os.RemoveAll(chapterPath)
 			downloadCount++
@@ -258,6 +251,8 @@ func downloadSeries(url string, downloadPath string, single float64) {
 		//if i == 5 && downloadCount < 6 {
 		//	break
 		//}
+
+		time.Sleep(1 * time.Second)
 	}
 
 	if downloadCount == 0 {