there we go

e621/download.go

@@ -0,0 +1,109 @@
+package e621
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/dustin/go-humanize"
+)
+
+// BeginDownload takes a slice of posts, a directory to save them in, and a
+// number of concurrent workers to make. It blocks until all the post have
+// been processed. It returns the number of successes, failures, and the total
+// amount of posts.
+func BeginDownload(posts *[]Post, saveDirectory *string, maxConcurrents *int) (*int, *int, *int) {
+	var wg sync.WaitGroup
+	var completed int
+	var successes int
+	var failures int
+
+	total := len(*posts)
+
+	// Distribute the posts based on the number of workers
+	ppw := len(*posts) / *maxConcurrents // ppw: posts per worker
+	mod := len(*posts) % *maxConcurrents // mod: remainder of posts
+
+	for i := 0; i < *maxConcurrents; i++ {
+		postsLower := i * ppw
+		postsUpper := i*ppw + ppw
+
+		if i == *maxConcurrents-1 {
+			// Give the last worker the remaining posts
+			// TODO: compensate it for labor
+			postsUpper += mod
+		}
+
+		wg.Add(1)
+		go work(i+1, (*posts)[postsLower:postsUpper], *saveDirectory, &completed, &successes, &failures, &total, &wg)
+		// Spawn workers with a little bit of a delay so as to not DDOS e621
+		// but also make the initial numbers show up correctly
+		time.Sleep(50 * time.Millisecond)
+	}
+
+	wg.Wait()
+
+	return &successes, &failures, &total
+}
+
+func work(wn int, posts []Post, directory string, completed *int, successes *int, failures *int, total *int, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	for _, post := range posts {
+		*completed++
+
+		fmt.Printf(
+			"[%d/%d] [w%d] Downloading post %d (%s) -> %s...\n",
+			*completed,
+			*total,
+			wn,
+			post.ID,
+			humanize.Bytes(uint64(post.FileSize)),
+			getSavePath(&post, &directory),
+		)
+
+		err := downloadPost(&post, directory)
+		if err != nil {
+			fmt.Printf("[w%d] Failed to download post %d: %v\n", wn, post.ID, err)
+			*failures++
+		} else {
+			*successes++
+		}
+	}
+}
+
+func getSavePath(post *Post, directory *string) string {
+	pathSliced := strings.Split(post.FileURL, ".")
+	extension := pathSliced[len(pathSliced)-1]
+
+	savePath := path.Join(*directory, strconv.Itoa(post.ID)+"."+extension)
+
+	return savePath
+}
+
+func downloadPost(post *Post, directory string) error {
+	savePath := getSavePath(post, &directory)
+
+	resp, err := HTTPGet(post.FileURL)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(savePath, body, 0755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

e621/e621.go

@@ -0,0 +1,87 @@
+package e621
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+)
+
+// Post represents an e621 post object returned by the e621 API.
+type Post struct {
+	ID             int            `json:"id"`
+	Tags           string         `json:"tags"`
+	LockedTags     bool           `json:"locked_tags"`
+	Description    string         `json:"description"`
+	CreatedAt      SerializedDate `json:"created_at"`
+	CreatorID      int            `json:"creator_id"`
+	Author         string         `json:"author"`
+	Change         int            `json:"change"`
+	Source         string         `json:"source"`
+	Score          int            `json:"score"`
+	FavoritesCount int            `json:"fav_count"`
+	MD5Hash        string         `json:"md5"`
+	FileSize       int            `json:"file_size"`
+	FileURL        string         `json:"file_url"`
+	FileExt        string         `json:"file_ext"`
+	PreviewURL     string         `json:"preview_url"`
+	PreviewHeight  int            `json:"preview_height"`
+	PreviewWidth   int            `json:"preview_width"`
+	Rating         string         `json:"rating"`
+	Status         string         `json:"status"`
+	Width          int            `json:"width"`
+	Height         int            `json:"height"`
+	HasComments    bool           `json:"has_comments"`
+	HasNotes       bool           `json:"has_notes"`
+	HasChildren    bool           `json:"has_children"`
+	Children       string         `json:"children"`
+	ParentID       int            `json:"parent_id"`
+	Artist         []string       `json:"artist"`
+	Sources        []string       `json:"sources"`
+}
+
+// SerializedDate represents a serialized date passed via JSON
+type SerializedDate struct {
+	JSONClass   string `json:"json_class"`
+	Seconds     int    `json:"s"`
+	Nanoseconds int    `json:"n"`
+}
+
+// GetPostsForTags gets a list of e621 Posts
+func GetPostsForTags(tags string, limit int, sfw bool) ([]Post, error) {
+	client := &http.Client{}
+
+	var domain string
+
+	if sfw {
+		domain = "e926.net"
+	} else {
+		domain = "e621.net"
+	}
+
+	req, _ := http.NewRequest("GET", "https://"+domain+"/post/index.json", nil)
+	req.Header.Set("User-Agent", "e6dl: go edition (@tjhorner on Telegram)")
+
+	qs := req.URL.Query()
+	qs.Add("tags", tags)
+	qs.Add("limit", strconv.Itoa(limit))
+
+	req.URL.RawQuery = qs.Encode()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var posts []Post
+	json.Unmarshal(body, &posts)
+
+	return posts, nil
+}

e621/util.go

@@ -0,0 +1,16 @@
+package e621
+
+import (
+	"net/http"
+)
+
+// HTTPGet is a helper function that automatically adds the
+// tool's UA to an HTTP GET request
+func HTTPGet(url string) (*http.Response, error) {
+	client := &http.Client{}
+
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("User-Agent", "e6dl: go edition (@tjhorner on Telegram)")
+
+	return client.Do(req)
+}