BitHeaven
/
r34dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add retrys

This commit is contained in:
BitHeaven 2023-03-06 19:39:23 +05:00
parent 90927b3c02
commit 8c247ff1c1
3 changed files with 37 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
go.mod
View File

@ -5,4 +5,10 @@ go 1.20
require (
github.com/logrusorgru/aurora v2.0.3+incompatible
golang.org/x/net v0.8.0
gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e
)
require (
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 // indirect
)

6
go.sum
View File

@ -1,4 +1,10 @@
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 h1:JAEbJn3j/FrhdWA9jW8B5ajsLIjeuEHLi8xE4fk997o=
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e h1:bJHzu9Qwc9wQRWJ/WVkJGAfs+riucl/tKAFNxf9pzqk=
gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e/go.mod h1:tve0rTLdGlwnXF7iBO9rbAEyeXvuuPx0n4DvXS/Nw7o=

34
main.go
View File

@ -18,6 +18,7 @@ import (
"github.com/logrusorgru/aurora"
"golang.org/x/net/proxy"
"gopkg.in/matryer/try.v1"
)
type Post struct {
@ -200,15 +201,30 @@ func work(wn int, state *workState, wc chan *Post, transport http.Transport) {
progress := aurora.Sprintf(aurora.Green("[%d/%d]"), state.Completed, state.Total)
workerText := aurora.Sprintf(aurora.Cyan("[w%d]"), wn)
fmt.Println(aurora.Sprintf(
"%s %s Downloading post %d -> %s",
progress,
workerText,
post.ID,
getSavePath(post, &state.SaveDirectory),
))
err := try.Do(func(attempt int) (retry bool, err error) {
if attempt == 1 {
fmt.Println(aurora.Sprintf(
"%s %s Downloading post %d -> %s",
progress,
workerText,
post.ID,
getSavePath(post, &state.SaveDirectory),
))
} else {
fmt.Println(aurora.Sprintf(
"%s %s (%d) Retry Download post %d -> %s",
progress,
workerText,
attempt,
post.ID,
getSavePath(post, &state.SaveDirectory),
))
}
err = downloadPost(post, state.SaveDirectory, transport)
return attempt < 5, err
})
err := downloadPost(post, state.SaveDirectory, transport)
if err != nil {
fmt.Printf("[w%d] Failed to download post %d: %v\n", wn, post.ID, err)
state.Failures++
@ -268,7 +284,7 @@ func main() {
tags := flag.String("tags", "", "Tags to search for")
maxConcurrents := flag.Int("concurrents", 30, "Maximum amount of concurrent downloads")
saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to")
postLimit := flag.Int("limit", 9999999999, "Maximum amount of posts to grab from e621")
postLimit := flag.Int("limit", 9999999999, "Maximum amount of posts to grab from rule34")
proxyAddr := flag.String("proxy", "", "Proxy address to parsing")
timeout := flag.Int("timeout", 10, "Timeout proxy to parsing")