This commit is contained in:
Itsigo
27
.gitea/workflows/build.yml
Normal file
27
.gitea/workflows/build.yml
Normal file
@@ -0,0 +1,27 @@
|
||||
name: Build this app
|
||||
run-name: ${{ gitea.actor }} is building GOMAD 🚀
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
Builder:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.26'
|
||||
|
||||
- name: Clone the repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build the linux binary
|
||||
run: make build
|
||||
|
||||
- name: Tar binary
|
||||
run: tar -czf ./bin/gomad.tar.gz ./tmp/gomad_linux_amd64
|
||||
|
||||
- name: Release
|
||||
uses: akkuman/gitea-release-action@v1
|
||||
with:
|
||||
files: |-
|
||||
bin/**
|
||||
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
downloads/
|
||||
tmp/
|
||||
manga.txt
|
||||
2
Makefile
Normal file
2
Makefile
Normal file
@@ -0,0 +1,2 @@
|
||||
build:
|
||||
OOS=linux GOARCH=amd64 go build -o tmp/gomad_linux_amd64
|
||||
25
README.md
25
README.md
@@ -0,0 +1,25 @@
|
||||
# GOMAD
|
||||
|
||||
**The Go Manga Downloader.**
|
||||
|
||||
A little CLI application to dowload Manga from WeebCentral. Designed to be run with cron.
|
||||
It it downloads all chapters and checks if there is a new chapter to be downloaded.
|
||||
This way I can have outomated downloads for the newest chapters.
|
||||
|
||||
Chapters will be compressed into a .cbz file. Since that's what Komnga supports ans I don't need more than that.
|
||||
|
||||
Usage:
|
||||
|
||||
```bash
|
||||
# Basic usage. Downloads the files into the current directory
|
||||
./gomad url=https://weebcentral.com/series/01JR0CSS15K1X2JG3YREKEY8AW/lovebullet
|
||||
|
||||
# You can provide a download path
|
||||
./gomad -path=downloads/ url=https://weebcentral.com/series/01JR0CSS15K1X2JG3YREKEY8AW/lovebullet
|
||||
|
||||
# You can provide a .txt file for multi import
|
||||
./gomad -path=downloads/ -file=manga.txt
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
33
go.mod
Normal file
33
go.mod
Normal file
@@ -0,0 +1,33 @@
|
||||
module git.itsigo.dev/itsigo/gomad
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.11.0
|
||||
github.com/gocolly/colly/v2 v2.3.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/antchfx/htmlquery v1.3.5 // indirect
|
||||
github.com/antchfx/xmlquery v1.5.0 // indirect
|
||||
github.com/antchfx/xpath v1.3.5 // indirect
|
||||
github.com/bits-and-blooms/bitset v1.24.4 // indirect
|
||||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/nlnwa/whatwg-url v0.6.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/schollz/progressbar/v3 v3.19.0 // indirect
|
||||
github.com/stretchr/testify v1.9.0 // indirect
|
||||
github.com/temoto/robotstxt v1.1.2 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/term v0.40.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
google.golang.org/protobuf v1.36.10 // indirect
|
||||
)
|
||||
137
go.sum
Normal file
137
go.sum
Normal file
@@ -0,0 +1,137 @@
|
||||
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
|
||||
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0=
|
||||
github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA=
|
||||
github.com/antchfx/xmlquery v1.5.0 h1:uAi+mO40ZWfyU6mlUBxRVvL6uBNZ6LMU4M3+mQIBV4c=
|
||||
github.com/antchfx/xmlquery v1.5.0/go.mod h1:lJfWRXzYMK1ss32zm1GQV3gMIW/HFey3xDZmkP1SuNc=
|
||||
github.com/antchfx/xpath v1.3.5 h1:PqbXLC3TkfeZyakF5eeh3NTWEbYl4VHNVeufANzDbKQ=
|
||||
github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/bits-and-blooms/bitset v1.24.4 h1:95H15Og1clikBrKr/DuzMXkQzECs1M6hhoGXLwLQOZE=
|
||||
github.com/bits-and-blooms/bitset v1.24.4/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
github.com/gocolly/colly/v2 v2.3.0 h1:HSFh0ckbgVd2CSGRE+Y/iA4goUhGROJwyQDCMXGFBWM=
|
||||
github.com/gocolly/colly/v2 v2.3.0/go.mod h1:Qp54s/kQbwCQvFVx8KzKCSTXVJ1wWT4QeAKEu33x1q8=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/nlnwa/whatwg-url v0.6.2 h1:jU61lU2ig4LANydbEJmA2nPrtCGiKdtgT0rmMd2VZ/Q=
|
||||
github.com/nlnwa/whatwg-url v0.6.2/go.mod h1:x0FPXJzzOEieQtsBT/AKvbiBbQ46YlL6Xa7m02M1ECk=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
|
||||
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
|
||||
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
274
main.go
Normal file
274
main.go
Normal file
@@ -0,0 +1,274 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/gocolly/colly/v2"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
)
|
||||
|
||||
type pageInfo struct {
|
||||
StatusCode int
|
||||
Links map[string]int
|
||||
}
|
||||
|
||||
type series struct {
|
||||
Name string
|
||||
Chapters []chapter
|
||||
}
|
||||
|
||||
type chapter struct {
|
||||
Name string
|
||||
Link string
|
||||
}
|
||||
|
||||
func extractSeriesInfo(url string) (string, string) {
|
||||
// Pattern: /series/{ID}/{name}
|
||||
c := colly.NewCollector()
|
||||
|
||||
var name string
|
||||
c.OnHTML("#top", func(e *colly.HTMLElement) {
|
||||
name = e.DOM.Find("h1").First().Text()
|
||||
})
|
||||
|
||||
c.Visit(url)
|
||||
re := regexp.MustCompile(`/series/([^/]+)`)
|
||||
matches := re.FindStringSubmatch(url)
|
||||
if len(matches) < 2 {
|
||||
return "", name
|
||||
}
|
||||
|
||||
return matches[1], name
|
||||
}
|
||||
|
||||
func FormatChapterString(s string) string {
|
||||
parts := strings.SplitN(s, ".", 2)
|
||||
intPart := parts[0]
|
||||
for len(intPart) < 3 {
|
||||
intPart = "0" + intPart
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
return intPart
|
||||
}
|
||||
return intPart + "." + parts[1]
|
||||
}
|
||||
|
||||
func getAllChapters(seriesID string) []chapter {
|
||||
url := fmt.Sprintf("https://weebcentral.com/series/%s/full-chapter-list", seriesID)
|
||||
|
||||
c := colly.NewCollector()
|
||||
|
||||
p := &pageInfo{Links: make(map[string]int)}
|
||||
|
||||
chapt := []chapter{}
|
||||
|
||||
count := 0
|
||||
// count links
|
||||
c.OnHTML("div", func(e *colly.HTMLElement) {
|
||||
|
||||
e.DOM.Find("a").Each(func(i int, h *goquery.Selection) {
|
||||
link, _ := h.Attr("href")
|
||||
name := strings.Split(h.Find("span span").First().Text(), " ")[1]
|
||||
chNum := FormatChapterString(name)
|
||||
chapt = append(chapt, chapter{
|
||||
Name: "Chapter " + chNum,
|
||||
Link: link,
|
||||
})
|
||||
count++
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
// extract status code
|
||||
c.OnResponse(func(r *colly.Response) {
|
||||
log.Println("response received", r.StatusCode)
|
||||
p.StatusCode = r.StatusCode
|
||||
})
|
||||
c.OnError(func(r *colly.Response, err error) {
|
||||
log.Println("error:", r.StatusCode, err)
|
||||
p.StatusCode = r.StatusCode
|
||||
})
|
||||
|
||||
c.Visit(url)
|
||||
|
||||
//slices.Reverse(chapt)
|
||||
return chapt
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
func downloadImage(url string, path string) {
|
||||
|
||||
c := colly.NewCollector()
|
||||
|
||||
c.OnResponse(func(r *colly.Response) {
|
||||
defer wg.Done()
|
||||
fullFileName := strings.Split(r.FileName(), "_")
|
||||
newFileName := fullFileName[len(fullFileName)-1]
|
||||
r.Save(path + newFileName)
|
||||
})
|
||||
c.Visit(url)
|
||||
}
|
||||
|
||||
func downloadChapter(chapterPath string, url string) {
|
||||
os.Mkdir(chapterPath, 0775)
|
||||
c := colly.NewCollector()
|
||||
count := 0
|
||||
c.OnHTML("section", func(h *colly.HTMLElement) {
|
||||
images := h.DOM.Find("img")
|
||||
imagesLength := images.Length()
|
||||
bar := progressbar.NewOptions(imagesLength,
|
||||
progressbar.OptionEnableColorCodes(true),
|
||||
progressbar.OptionFullWidth(),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionSetDescription("[cyan][1/2][reset] Downloading"),
|
||||
progressbar.OptionSetTheme(progressbar.Theme{
|
||||
Saucer: "[green]=[reset]",
|
||||
SaucerHead: "[green]>[reset]",
|
||||
SaucerPadding: " ",
|
||||
BarStart: "[",
|
||||
BarEnd: "]",
|
||||
}))
|
||||
images.Each(func(i int, s *goquery.Selection) {
|
||||
imgSrc, _ := s.Attr("src")
|
||||
bar.Add(1)
|
||||
wg.Add(1)
|
||||
count++
|
||||
go downloadImage(imgSrc, chapterPath+"/")
|
||||
// with this I can download 10 images at the same time
|
||||
if count == 10 || i == imagesLength-1 {
|
||||
wg.Wait()
|
||||
count = 0
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
c.Visit(fmt.Sprint(url, "/images?is_prev=False¤t_page=1&reading_style=long_strip"))
|
||||
|
||||
}
|
||||
|
||||
func compressToCbz(chapterPath string) {
|
||||
path := chapterPath + ".cbz"
|
||||
archive, err := os.Create(path)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer archive.Close()
|
||||
|
||||
// Create a new zip writer
|
||||
zipWriter := zip.NewWriter(archive)
|
||||
|
||||
dir, err := os.ReadDir(chapterPath)
|
||||
|
||||
bar := progressbar.NewOptions(len(dir),
|
||||
progressbar.OptionEnableColorCodes(true),
|
||||
progressbar.OptionFullWidth(),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionSetDescription("[magenta][2/2][reset] Compressing"),
|
||||
progressbar.OptionSetTheme(progressbar.Theme{
|
||||
Saucer: "[green]=[reset]",
|
||||
SaucerHead: "[green]>[reset]",
|
||||
SaucerPadding: " ",
|
||||
BarStart: "[",
|
||||
BarEnd: "]",
|
||||
}))
|
||||
|
||||
for _, e := range dir {
|
||||
f1, err := os.Open(chapterPath + "/" + e.Name())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
bar.Add(1)
|
||||
defer f1.Close()
|
||||
|
||||
w1, err := zipWriter.Create(e.Name())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := io.Copy(w1, f1); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
}
|
||||
fmt.Println()
|
||||
zipWriter.Close()
|
||||
}
|
||||
|
||||
func downloadSeries(url string, downloadPath string) {
|
||||
log.Println("Downloading from", url)
|
||||
|
||||
seriesID, seriesName := extractSeriesInfo(url)
|
||||
|
||||
s := series{}
|
||||
|
||||
s.Name = seriesName
|
||||
s.Chapters = getAllChapters(seriesID)
|
||||
log.Printf("Found %d Chapters\n", len(s.Chapters))
|
||||
|
||||
seriesPath := fmt.Sprintf("%s%s", downloadPath, seriesName)
|
||||
|
||||
os.Mkdir(seriesPath, 0775)
|
||||
|
||||
log.Println("Starting to download", seriesName)
|
||||
downloadCount := 0
|
||||
for i, c := range s.Chapters {
|
||||
chapterPath := fmt.Sprintf("%s/%s", seriesPath, c.Name)
|
||||
if _, err := os.Stat(seriesPath + "/" + c.Name); os.IsNotExist(err) {
|
||||
log.Println("Downloading ", c.Name)
|
||||
downloadChapter(chapterPath, c.Link)
|
||||
//log.Println("Compressing", c.Name)
|
||||
compressToCbz(chapterPath)
|
||||
os.RemoveAll(chapterPath)
|
||||
downloadCount++
|
||||
}
|
||||
|
||||
// if after 5 five files no download has happened just move on to the next
|
||||
if i == 5 && downloadCount < 6 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if downloadCount == 0 {
|
||||
log.Println("No new Chapters downloaded")
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
urlPtr := flag.String("url", "", "WeebCentral URL")
|
||||
dlPathPtr := flag.String("path", "", "Download Path")
|
||||
filePtr := flag.String("file", "", "Path to file for multi download")
|
||||
|
||||
flag.Parse()
|
||||
if *urlPtr == "" && *filePtr == "" {
|
||||
fmt.Println("Please use a url to use this command")
|
||||
os.Exit(1)
|
||||
}
|
||||
//if *dlPathPtr == "" {
|
||||
// fmt.Println("Please provide a path to download the files to")
|
||||
// os.Exit(1)
|
||||
//}
|
||||
|
||||
if *filePtr != "" {
|
||||
dat, err := os.ReadFile(*filePtr)
|
||||
if err != nil {
|
||||
fmt.Println("Could not find file")
|
||||
}
|
||||
paths := strings.Split(strings.TrimRight(string(dat), "\n"), "\n")
|
||||
|
||||
for _, p := range paths {
|
||||
downloadSeries(p, *dlPathPtr)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
downloadSeries(*urlPtr, *dlPathPtr)
|
||||
}
|
||||
Reference in New Issue
Block a user