BitHeaven
/
r34dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add check exist files

This commit is contained in:
BitHeaven 2023-03-06 15:40:46 +05:00 committed by GitHub
parent b008b67bcb
commit 4cc6c079b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 17 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
main.go
View File

@ -229,21 +229,25 @@ func getSavePath(post *Post, directory *string) string {
func downloadPost(post *Post, directory string, transport http.Transport) error { func downloadPost(post *Post, directory string, transport http.Transport) error {
savePath := getSavePath(post, &directory) savePath := getSavePath(post, &directory)
resp, err := HTTPGet(post.FileURL, transport) if _, err := os.Stat(savePath); err == nil {
if err != nil { fmt.Print("File exists, skip...\n")
return err } else {
} resp, err := HTTPGet(post.FileURL, transport)
if err != nil {
return err
}
defer resp.Body.Close() defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return err return err
} }
err = ioutil.WriteFile(savePath, body, 0755) err = ioutil.WriteFile(savePath, body, 0755)
if err != nil { if err != nil {
return err return err
}
} }
return nil return nil
@ -264,7 +268,7 @@ func main() {
tags := flag.String("tags", "", "Tags to search for") tags := flag.String("tags", "", "Tags to search for")
maxConcurrents := flag.Int("concurrents", 30, "Maximum amount of concurrent downloads") maxConcurrents := flag.Int("concurrents", 30, "Maximum amount of concurrent downloads")
saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to") saveDirectory := flag.String("out", "dl", "The directory to write the downloaded posts to")
postLimit := flag.Int("limit", 1000, "Maximum amount of posts to grab from e621") postLimit := flag.Int("limit", 9999999999, "Maximum amount of posts to grab from e621")
proxyAddr := flag.String("proxy", "", "Proxy address to parsing") proxyAddr := flag.String("proxy", "", "Proxy address to parsing")
timeout := flag.Int("timeout", 10, "Timeout proxy to parsing") timeout := flag.Int("timeout", 10, "Timeout proxy to parsing")