package main import ( "archive/zip" "flag" "fmt" "io" "log" "os" "regexp" "strings" "sync" "time" "github.com/PuerkitoBio/goquery" "github.com/schollz/progressbar/v3" "gopkg.in/headzoo/surf.v1" ) var simplePtr *bool type pageInfo struct { StatusCode int Links map[string]int } type series struct { Name string Chapters []chapter } type chapter struct { Name string Link string } func extractSeriesInfo(url string) (string, string) { // Pattern: /series/{ID}/{name} bow := surf.NewBrowser() bow.Open(url) var name string name = bow.Find("h1").First().Text() re := regexp.MustCompile(`/series/([^/]+)`) matches := re.FindStringSubmatch(url) if len(matches) < 2 { return "", name } return matches[1], name } func FormatChapterString(s string) string { parts := strings.SplitN(s, ".", 2) intPart := parts[0] for len(intPart) < 3 { intPart = "0" + intPart } if len(parts) == 1 { return intPart } return intPart + "." + parts[1] } func getAllChapters(seriesID string, single float64) []chapter { url := fmt.Sprintf("https://weebcentral.com/series/%s/full-chapter-list", seriesID) bow := surf.NewBrowser() err := bow.Open(url) if err != nil { panic(err) } chapt := []chapter{} bow.Find("div a").Each(func(i int, h *goquery.Selection) { link, _ := h.Attr("href") name := strings.Split(h.Find("span span").First().Text(), " ")[1] chNum := FormatChapterString(name) if single != 9999 { if name == fmt.Sprintf("%g", single) { chapt = append(chapt, chapter{ Name: "Chapter " + chNum, Link: link, }) } } else { chapt = append(chapt, chapter{ Name: "Chapter " + chNum, Link: link, }) } }) return chapt } var wg sync.WaitGroup func downloadImage(url string, path string) { bow := surf.NewBrowser() err := bow.Open(url) if err != nil { log.Fatal(err) } fullFileName := strings.Split(url, "-") newFileName := fullFileName[len(fullFileName)-1] //open a file for writing file, err := os.Create(path + newFileName) if err != nil { log.Fatal(err) } bow.Download(file) } func downloadChapter(chapterPath string, url string) { os.Mkdir(chapterPath, 0775) bow := surf.NewBrowser() err := bow.Open(url) if err != nil { panic(err) } images := bow.Images() imagesLength := len(images) //open a file for writing count := 0 bar := progressbar.NewOptions(imagesLength, progressbar.OptionEnableColorCodes(true), progressbar.OptionFullWidth(), progressbar.OptionShowCount(), progressbar.OptionSetDescription("[cyan][1/2][reset] Downloading Chapter"), progressbar.OptionSetTheme(progressbar.Theme{ Saucer: "[green]=[reset]", SaucerHead: "[green]>[reset]", SaucerPadding: " ", BarStart: "[", BarEnd: "]", })) for _, img := range images { imgSrc := img.Url().String() if !*simplePtr { bar.Add(1) } wg.Add(1) count++ downloadImage(imgSrc, chapterPath+"/") // with this I can download 10 images at the same time //if count == 10 || i == imagesLength-1 { // wg.Wait() // count = 0 //} } } func compressToCbz(chapterPath string) { path := chapterPath + ".cbz" archive, err := os.Create(path) if err != nil { panic(err) } defer archive.Close() // Create a new zip writer zipWriter := zip.NewWriter(archive) dir, err := os.ReadDir(chapterPath) bar := progressbar.NewOptions(len(dir), progressbar.OptionEnableColorCodes(true), progressbar.OptionFullWidth(), progressbar.OptionShowCount(), progressbar.OptionSetDescription("[magenta][2/2][reset] Compressing"), progressbar.OptionSetTheme(progressbar.Theme{ Saucer: "[green]=[reset]", SaucerHead: "[green]>[reset]", SaucerPadding: " ", BarStart: "[", BarEnd: "]", })) for _, e := range dir { f1, err := os.Open(chapterPath + "/" + e.Name()) if err != nil { panic(err) } if !*simplePtr { bar.Add(1) } defer f1.Close() w1, err := zipWriter.Create(e.Name()) if err != nil { panic(err) } if _, err := io.Copy(w1, f1); err != nil { panic(err) } } fmt.Println() zipWriter.Close() } func downloadSeries(url string, downloadPath string, single float64) { log.Println("Downloading from", url) seriesID, seriesName := extractSeriesInfo(url) s := series{} s.Name = seriesName s.Chapters = getAllChapters(seriesID, single) log.Printf("Found %d Chapters\n", len(s.Chapters)) seriesPath := fmt.Sprintf("%s%s", downloadPath, seriesName) os.Mkdir(seriesPath, 0775) log.Println("Starting to download", seriesName) downloadCount := 0 for _, c := range s.Chapters { chapterPath := fmt.Sprintf("%s/%s", seriesPath, c.Name) if _, err := os.Stat(seriesPath + "/" + c.Name + ".cbz"); os.IsNotExist(err) { log.Println("Downloading ", c.Name) downloadChapter(chapterPath, "https://weebcentral.com"+c.Link+"/images") log.Println("Compressing", c.Name) compressToCbz(chapterPath) os.RemoveAll(chapterPath) downloadCount++ } // if after 5 five files no download has happened just move on to the next //if i == 5 && downloadCount < 6 { // break //} time.Sleep(1 * time.Second) } if downloadCount == 0 { log.Println("No new Chapters downloaded") } } func main() { urlPtr := flag.String("url", "", "WeebCentral URL") dlPathPtr := flag.String("path", "", "Download Path") filePtr := flag.String("file", "", "Path to file for multi download") chapterPtr := flag.Float64("c", 9999, "Specific chapter") simplePtr = flag.Bool("s", false, "Simple output for readable logs") flag.Parse() if *urlPtr == "" && *filePtr == "" { fmt.Println("Please use a url to use this command") os.Exit(1) } //if *dlPathPtr == "" { // fmt.Println("Please provide a path to download the files to") // os.Exit(1) //} if *filePtr != "" { dat, err := os.ReadFile(*filePtr) if err != nil { fmt.Println("Could not find file") } paths := strings.SplitSeq(strings.TrimRight(string(dat), "\n"), "\n") for p := range paths { downloadSeries(p, *dlPathPtr, *chapterPtr) } os.Exit(0) } downloadSeries(*urlPtr, *dlPathPtr, *chapterPtr) }