diff --git a/go.mod b/go.mod index f6d0aac..7f9f406 100644 --- a/go.mod +++ b/go.mod @@ -5,4 +5,10 @@ go 1.20 require ( github.com/logrusorgru/aurora v2.0.3+incompatible golang.org/x/net v0.8.0 + gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e +) + +require ( + github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect + github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 // indirect ) diff --git a/go.sum b/go.sum index 943442d..346f4b1 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,10 @@ +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 h1:JAEbJn3j/FrhdWA9jW8B5ajsLIjeuEHLi8xE4fk997o= +github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e h1:bJHzu9Qwc9wQRWJ/WVkJGAfs+riucl/tKAFNxf9pzqk= +gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e/go.mod h1:tve0rTLdGlwnXF7iBO9rbAEyeXvuuPx0n4DvXS/Nw7o= diff --git a/main.go b/main.go index 15ee7e7..0265aae 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "github.com/logrusorgru/aurora" "golang.org/x/net/proxy" + "gopkg.in/matryer/try.v1" ) type Post struct { @@ -200,15 +201,30 @@ func work(wn int, state *workState, wc chan *Post, transport http.Transport) { progress := aurora.Sprintf(aurora.Green("[%d/%d]"), state.Completed, state.Total) workerText := aurora.Sprintf(aurora.Cyan("[w%d]"), wn) - fmt.Println(aurora.Sprintf( - "%s %s Downloading post %d -> %s", - progress, - workerText, - post.ID, - getSavePath(post, &state.SaveDirectory), - )) + err := try.Do(func(attempt int) (retry bool, err error) { + if attempt == 1 { + fmt.Println(aurora.Sprintf( + "%s %s Downloading post %d -> %s", + progress, + workerText, + post.ID, + getSavePath(post, &state.SaveDirectory), + )) + } else { + fmt.Println(aurora.Sprintf( + "%s %s (%d) Retry Download post %d -> %s", + progress, + workerText, + attempt, + post.ID, + getSavePath(post, &state.SaveDirectory), + )) + } + + err = downloadPost(post, state.SaveDirectory, transport) + return attempt < 5, err + }) - err := downloadPost(post, state.SaveDirectory, transport) if err != nil { fmt.Printf("[w%d] Failed to download post %d: %v\n", wn, post.ID, err) state.Failures++ @@ -268,7 +284,7 @@ func main() { tags := flag.String("tags", "", "Tags to search for") maxConcurrents := flag.Int("concurrents", 30, "Maximum amount of concurrent downloads") saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to") - postLimit := flag.Int("limit", 9999999999, "Maximum amount of posts to grab from e621") + postLimit := flag.Int("limit", 9999999999, "Maximum amount of posts to grab from rule34") proxyAddr := flag.String("proxy", "", "Proxy address to parsing") timeout := flag.Int("timeout", 10, "Timeout proxy to parsing")