implement cursoring for timelines

drivel.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
@@ -20,6 +21,8 @@ const (
 	CHARACTER_LIMIT         = 280
 	WIPE_KEEP_DAYS          = 10
 	STATUS_ENDPOINT         = "https://api.twitter.com/1.1/statuses/update.json"
+	MAX_TIMELINE_REQUESTS   = 15
+	DEFAULT_COUNT           = 200
 	MENTIONS_ENDPOINT       = "https://api.twitter.com/1.1/statuses/mentions_timeline.json?tweet_mode=extended&count=200"
 	HOME_ENDPOINT           = "https://api.twitter.com/1.1/statuses/home_timeline.json?tweet_mode=extended&count=200"
 	TIMELINE_ENDPOINT       = "https://api.twitter.com/1.1/statuses/user_timeline.json?tweet_mode=extended&count=200"
@@ -33,8 +36,7 @@ const (
 
 func optLogFatal(decorum string, err error) {
 	if err != nil && err.Error() != "" {
-		fmt.Fprintln(os.Stderr, "drivel: "+decorum+": "+err.Error())
+		panic("drivel: " + decorum + ": " + err.Error())
-		os.Exit(-1)
 	}
 }
 
@@ -70,9 +72,7 @@ func _send(url string, vals url.Values, usePost bool) []byte {
 	defer resp.Body.Close()
 	body, err := ioutil.ReadAll(resp.Body)
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		fmt.Fprintln(os.Stderr, "response:", resp, "\n")
+		log(errors.New(fmt.Sprintf("HTTP status %d\n\nresponse: %v\n\nbody: %s", resp.StatusCode, resp, string(body))))
-		fmt.Fprintln(os.Stderr, "body:", string(body), "\n")
-		log(errors.New("HTTP status " + fmt.Sprint(resp.StatusCode)))
 	}
 	log(err)
 	return body
@@ -289,8 +289,11 @@ func lookup(args []string) error {
 	return nil
 }
 
-func timeline(endpoint string) []Status {
+func timeline(endpoint string, maxID string) []Status {
 	log := func(err error) { optLogFatal("timeline", err) }
+	if maxID != "" {
+		endpoint += "&max_id=" + maxID
+	}
 	body := get(endpoint)
 	var tweets []Status
 	err := json.Unmarshal(body, &tweets)
@@ -298,34 +301,86 @@ func timeline(endpoint string) []Status {
 	return tweets
 }
 
-func mentions(args []string) error {
+func timelineLoop(endpoint string, flags timelineFlags) (tweets []Status) {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Fprintln(os.Stderr, "INFO:", r)
+		}
+	}()
+	requestCount := 0
+	maxID := ""
+	for len(tweets) < flags.count && requestCount < flags.maxRequests {
+		tmp := timeline(endpoint, maxID)
+		if len(tmp) == 0 {
+			break
+		}
+		lowestSoFar, _ := strconv.Atoi(tmp[len(tmp)-1].Id_str)
+		maxID = strconv.Itoa(lowestSoFar - 1)
+		tweets = append(tweets, tmp...)
+		if len(tweets) > flags.count {
+			tweets = tweets[:flags.count]
+		}
+		requestCount++
+	}
+	return
+}
+
+type timelineFlags struct {
+	count       int
+	maxRequests int
+}
+
+func timelineFlagsVars(s *flag.FlagSet, f *timelineFlags) {
+	s.IntVar(&f.count, "count", DEFAULT_COUNT, "try to get up to N tweets")
+	s.IntVar(&f.maxRequests, "max-requests", MAX_TIMELINE_REQUESTS, "try to achieve count with a maximum of N requests")
+}
+
+func mentions(flags timelineFlags, args []string) error {
 	checkUsage(args, 0, 0, "mentions")
-	tweets := timeline(MENTIONS_ENDPOINT)
+	tweets := timelineLoop(MENTIONS_ENDPOINT, flags)
 	PrintTweets(tweets, mentionsFilter)
 	return nil
 }
 
-func home(args []string) error {
+func mentionsCommand() (goutil.CommandFlagsInit, goutil.CommandFunc) {
+	f := timelineFlags{}
+	flagsInit := func(s *flag.FlagSet) {
+		timelineFlagsVars(s, &f)
+	}
+	return flagsInit, func(args []string) error { return mentions(f, args) }
+}
+
+func home(flags timelineFlags, args []string) error {
 	checkUsage(args, 0, 0, "home")
-	tweets := timeline(HOME_ENDPOINT)
+	tweets := timelineLoop(HOME_ENDPOINT, flags)
 	PrintTweets(tweets, homeFilter)
 	return nil
 }
 
+func homeCommand() (goutil.CommandFlagsInit, goutil.CommandFunc) {
+	f := timelineFlags{}
+	flagsInit := func(s *flag.FlagSet) {
+		timelineFlagsVars(s, &f)
+	}
+	return flagsInit, func(args []string) error { return home(f, args) }
+}
+
 func userTimeline(flags userTimelineFlags, args []string) error {
 	checkUsage(args, 1, 1, "timeline USER")
-	tweets := timeline(TIMELINE_ENDPOINT + UserTimelineParameters(flags, args[0]))
+	tweets := timelineLoop(TIMELINE_ENDPOINT+UserTimelineParameters(flags, args[0]), flags.timelineFlags)
 	PrintTweets(tweets, nil)
 	return nil
 }
 
 type userTimelineFlags struct {
+	timelineFlags
 	withReplies bool
 }
 
 func userTimelineCommand() (goutil.CommandFlagsInit, goutil.CommandFunc) {
 	f := userTimelineFlags{}
 	flagsInit := func(s *flag.FlagSet) {
+		timelineFlagsVars(s, &f.timelineFlags)
 		s.BoolVar(&f.withReplies, "with-replies", false, "include replies in timeline")
 	}
 	return flagsInit, func(args []string) error { return userTimeline(f, args) }
@@ -385,7 +440,8 @@ func wipeTimeline(likes bool, keepDays int) {
 	}
 	n := 0
 	now := time.Now()
-	tweets := timeline(endpoint)
+	maxID := ""
+	tweets := timeline(endpoint, maxID)
 	for {
 		for _, tweet := range tweets {
 			daysSince := now.Sub(tweet.Created_at.Time).Hours() / 24
@@ -401,8 +457,9 @@ func wipeTimeline(likes bool, keepDays int) {
 					return
 				}
 			}
+			maxID = tweet.Id_str
 		}
-		newTweets := timeline(endpoint)
+		newTweets := timeline(endpoint, maxID)
 		if !equals(newTweets, tweets) {
 			tweets = newTweets
 		} else {
@@ -504,12 +561,18 @@ var mentionsFilter hashset
 var formatTemplate *template.Template
 
 func main() {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Fprintln(os.Stderr, r)
+			os.Exit(-1)
+		}
+	}()
 	client = getClient()
 	setFilters(appDir())
 	commands := []goutil.Command{
 		goutil.NewCommandWithFlags("status", wrapCommand(status), "post a status with message and/or media"),
-		goutil.NewCommandWithFlags("home", wrapCommand(home), "get your home timeline"),
+		goutil.NewCommandWithFlags("home", wrapCommandFl(homeCommand), "get your home timeline"),
-		goutil.NewCommandWithFlags("mentions", wrapCommand(mentions), "get your mention timeline"),
+		goutil.NewCommandWithFlags("mentions", wrapCommandFl(mentionsCommand), "get your mention timeline"),
 		goutil.NewCommandWithFlags("timeline", wrapCommandFl(userTimelineCommand), "get timeline of a specific user"),
 		goutil.NewCommandWithFlags("lookup", wrapCommand(lookup), "lookup tweets with specific IDs"),
 		goutil.NewCommandWithFlags("reply", wrapCommand(reply), "reply to a tweet with a specific ID"),
@@ -518,9 +581,5 @@ func main() {
 		goutil.NewCommandWithFlags("like", wrapCommand(like), "like a tweet with a specific ID"),
 		goutil.NewCommandWithFlags("wipe", wipeCommand, "wipe your timeline and likes"),
 	}
-	err := goutil.Execute(commands)
+	_ = goutil.Execute(commands)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
 }