e6dl/main.go

63 lines
1.5 KiB
Go
Raw Normal View History

2019-03-01 03:47:50 +08:00
package main
import (
"flag"
"fmt"
"os"
"path"
2019-03-14 03:16:40 +08:00
2022-11-10 23:00:48 +08:00
"github.com/BitHeaven-Official/e6dl/concurrent"
"github.com/BitHeaven-Official/e6dl/e621"
2019-03-01 03:47:50 +08:00
)
func main() {
// define cmd line flags
tags := flag.String("tags", "", "Tags to search for")
maxConcurrents := flag.Int("concurrents", 5, "Maximum amount of concurrent downloads")
2022-11-10 22:58:56 +08:00
postLimit := flag.Int("limit", 99999999, "Maximum amount of posts to grab from e621")
saveDirectory := flag.String("out", "e621", "The directory to write the downloaded posts to")
sfw := flag.Bool("sfw", false, "Download posts from e926 instead of e621")
2019-03-01 03:47:50 +08:00
flag.Parse()
2022-11-10 22:58:56 +08:00
fmt.Printf("Fetching posts for \"%s\" (limit=%d)\n", *tags, *postLimit)
2019-03-01 03:47:50 +08:00
2019-09-06 10:05:44 +08:00
var allPosts []e621.Post
2022-11-10 22:58:56 +08:00
i := 1
for {
2019-09-06 10:05:44 +08:00
fmt.Printf("Fetching page %d/%d...", i, *pages)
posts, err := e621.GetPostsForTags(*tags, *postLimit, *sfw, i)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Printf(" fetched %d posts\n", len(posts))
2022-11-10 22:58:56 +08:00
if len(posts) == 0 {
break
}
2019-09-06 10:05:44 +08:00
allPosts = append(allPosts, posts...)
2022-11-10 22:58:56 +08:00
i++
2019-03-01 03:47:50 +08:00
}
2019-09-06 10:05:44 +08:00
fmt.Printf("Found %d posts. Starting download with %d workers...\n\n", len(allPosts), *maxConcurrents)
2019-03-01 03:47:50 +08:00
cwd, _ := os.Getwd()
absSaveDir := path.Join(cwd, *saveDirectory)
2019-09-06 10:05:44 +08:00
err := os.MkdirAll(absSaveDir, 0755)
2019-03-01 03:47:50 +08:00
if err != nil {
fmt.Printf("Cannot create output directory (%s). Do you have the right permissions?\n", absSaveDir)
os.Exit(1)
}
2019-09-06 10:05:44 +08:00
successes, failures, _ := concurrent.BeginDownload(&allPosts, saveDirectory, maxConcurrents)
2019-03-01 03:47:50 +08:00
2019-03-01 08:22:36 +08:00
fmt.Printf("\nAll done! %d posts downloaded and saved. (%d failed to download)\n", *successes, *failures)
2019-03-01 03:47:50 +08:00
}