drivel/drivel.go
gutmet fd599da4bb flag '--with-replies' for timeline subcommand
* this changes its default behaviour to exclude replies
2020-10-15 14:47:28 +02:00

878 lines
20 KiB
Go

package main
import (
"encoding/base64"
"encoding/json"
"errors"
"flag"
"fmt"
goutil "git.gutmet.org/goutil.git/misc"
"html"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
)
const (
MAX_BYTES = 50 * 1024 * 1024
CHUNK_SIZE = 1024 * 1024
CHARACTER_LIMIT = 280
WIPE_KEEP_DAYS = 10
UPLOAD_ENDPOINT = "https://upload.twitter.com/1.1/media/upload.json"
STATUS_ENDPOINT = "https://api.twitter.com/1.1/statuses/update.json"
MENTIONS_ENDPOINT = "https://api.twitter.com/1.1/statuses/mentions_timeline.json?tweet_mode=extended&count=200"
HOME_ENDPOINT = "https://api.twitter.com/1.1/statuses/home_timeline.json?tweet_mode=extended&count=200"
TIMELINE_ENDPOINT = "https://api.twitter.com/1.1/statuses/user_timeline.json?tweet_mode=extended&count=200"
LIKES_TIMELINE_ENDPOINT = "https://api.twitter.com/1.1/favorites/list.json?tweet_mode=extended&count=200"
LOOKUP_ENDPOINT = "https://api.twitter.com/1.1/statuses/lookup.json?tweet_mode=extended"
RETWEET_ENDPOINT = "https://api.twitter.com/1.1/statuses/retweet/"
LIKE_ENDPOINT = "https://api.twitter.com/1.1/favorites/create.json"
DESTROY_STATUS_ENDPOINT = "https://api.twitter.com/1.1/statuses/destroy/"
DESTROY_LIKE_ENDPOINT = "https://api.twitter.com/1.1/favorites/destroy.json"
)
func optLogFatal(decorum string, err error) {
if err != nil && err.Error() != "" {
fmt.Fprintln(os.Stderr, "drivel: "+decorum+": "+err.Error())
os.Exit(-1)
}
}
type ObjectID string
type TwitterError struct {
Code int64
Message string
Label string
}
type Response struct {
Errors []TwitterError
}
func (r Response) Error() string {
if len(r.Errors) == 0 {
return ""
} else {
s, _ := json.Marshal(r)
return "Response error " + string(s)
}
}
func InitRequest(mediaType string, totalBytes int) url.Values {
r := map[string][]string{
"command": {"INIT"},
"media_type": {mediaType},
"total_bytes": {strconv.Itoa(totalBytes)},
}
if mediaType == "video/mp4" {
r["media_category"] = []string{"tweet_video"}
}
return r
}
type InitResponse struct {
Response
Media_id_string string
}
func AppendRequest(mediaID ObjectID, mediaData string, segmentIndex int) url.Values {
return map[string][]string{
"command": {"APPEND"},
"media_id": {string(mediaID)},
"media_data": {mediaData},
"segment_index": {strconv.Itoa(segmentIndex)},
}
}
func FinalizeRequest(mediaID ObjectID) url.Values {
return map[string][]string{
"command": {"FINALIZE"},
"media_id": {string(mediaID)},
}
}
type FinalizeResponse struct {
Error string
Media_id_string string
Processing_info ProcessingInfo
}
type ProcessingInfo struct {
State string
Check_after_secs int64
Progress_percent int64
Error TwitterError
}
func PollStatusParameters(mediaID ObjectID) string {
return "?command=STATUS&media_id=" + string(mediaID)
}
type PollStatusResponse struct {
Processing_info ProcessingInfo
}
func UpdateStatusRequest(status string, mediaIDs []ObjectID, previousStatusID ObjectID) url.Values {
r := map[string][]string{"status": {status}}
if len(mediaIDs) > 0 {
ids := []string{}
for _, id := range mediaIDs {
ids = append(ids, string(id))
}
r["media_ids"] = []string{strings.Join(ids, ",")}
}
if len(previousStatusID) > 0 {
r["in_reply_to_status_id"] = []string{string(previousStatusID)}
r["auto_populate_reply_metadata"] = []string{"true"}
}
return r
}
type UpdateStatusResponse struct {
Response
Id_str string
}
func LookupParameters(ids []string) string {
return "&id=" + strings.Join(ids, ",")
}
var mimetype = map[string]string{
".mp4": "video/mp4",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
}
func getMimetype(file string) string {
ext := filepath.Ext(file)
if v, ok := mimetype[ext]; ok {
return v
} else {
return "application/octet-stream"
}
}
func get(url string) []byte {
return _send(url, nil, false)
}
func send(url string, vals url.Values) []byte {
return _send(url, vals, true)
}
func _send(url string, vals url.Values, usePost bool) []byte {
log := func(err error) {
v, _ := json.Marshal(vals)
optLogFatal("get/post "+url+" "+string(v), err)
}
var resp *http.Response
var err error
if usePost {
resp, err = client.PostForm(url, vals)
} else {
resp, err = client.Get(url)
}
log(err)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if resp.StatusCode < 200 || resp.StatusCode > 299 {
fmt.Fprintln(os.Stderr, "response:", resp, "\n")
fmt.Fprintln(os.Stderr, "body:", string(body), "\n")
log(errors.New("HTTP status " + fmt.Sprint(resp.StatusCode)))
}
log(err)
return body
}
func initFileUpload(file string, mediaData []byte) ObjectID {
log := func(err error) { optLogFatal("initFileUpload "+file, err) }
initRequest := InitRequest(getMimetype(file), len(mediaData))
body := send(UPLOAD_ENDPOINT, initRequest)
var initResponse InitResponse
err := json.Unmarshal(body, &initResponse)
log(err)
log(initResponse)
return ObjectID(initResponse.Media_id_string)
}
func appendFileChunks(file string, media string, mediaId ObjectID) {
log := func(err error) { optLogFatal("appendFileChunks", err) }
info := func(v ...interface{}) {
if len(media) > CHUNK_SIZE {
fmt.Println(v...)
}
}
info("chunk upload", file)
info("total", len(media))
for i := 0; i*CHUNK_SIZE < len(media); i = i + 1 {
start := i * CHUNK_SIZE
end := (i + 1) * CHUNK_SIZE
if end > len(media) {
end = len(media)
}
info("segment", i, "start", start, "end", end)
appended := false
var body []byte
for try := 0; try < 3 && !appended; try++ {
appRequest := AppendRequest(mediaId, media[start:end], i)
body = send(UPLOAD_ENDPOINT, appRequest)
if string(body) == "" {
appended = true
}
}
if !appended {
log(errors.New(string(body)))
}
}
}
func finalizeFileUpload(file string, mediaId ObjectID) int64 {
log := func(err error) { optLogFatal("finalizeFileUpload", err) }
body := send(UPLOAD_ENDPOINT, FinalizeRequest(mediaId))
var finalizeResponse FinalizeResponse
err := json.Unmarshal(body, &finalizeResponse)
log(err)
log(errors.New(finalizeResponse.Error))
if id := ObjectID(finalizeResponse.Media_id_string); id != "" {
fmt.Println("==> Uploaded " + file + " with id " + string(id))
procInfo := finalizeResponse.Processing_info
return procInfo.Check_after_secs
} else {
log(errors.New("Could not finalize " + string(mediaId)))
return 0
}
}
func wait(seconds int64) {
fmt.Println("Waiting", seconds, "seconds")
time.Sleep(time.Duration(seconds) * time.Second)
}
func pollStatus(mediaId ObjectID) {
log := func(err error) { optLogFatal("pollStatus "+string(mediaId), err) }
succeeded := false
var error TwitterError
for try := 0; try < 6; try = try + 1 {
body := get(UPLOAD_ENDPOINT + PollStatusParameters(mediaId))
var response PollStatusResponse
err := json.Unmarshal(body, &response)
log(err)
procInfo := response.Processing_info
state := procInfo.State
error = procInfo.Error
if state == "succeeded" {
succeeded = true
break
} else if state == "failed" {
break
} else {
fmt.Println("Processing progress: ", procInfo.Progress_percent, "%")
seconds := procInfo.Check_after_secs
if seconds > 10 {
seconds = 10
}
wait(seconds)
}
}
if !succeeded {
log(errors.New("File upload failed " + error.Message))
}
}
func uploadFile(file string) ObjectID {
log := func(err error) { optLogFatal("uploadFile "+file, err) }
tmpMedia, err := ioutil.ReadFile(file)
log(err)
media := base64.RawURLEncoding.EncodeToString(tmpMedia)
mediaId := initFileUpload(file, tmpMedia)
appendFileChunks(file, media, mediaId)
seconds := finalizeFileUpload(file, mediaId)
if seconds > 0 {
wait(seconds)
pollStatus(mediaId)
}
return mediaId
}
func uploadAll(files []string) []ObjectID {
ids := []ObjectID{}
for _, f := range files {
if f != "" {
id := uploadFile(f)
ids = append(ids, id)
}
}
return ids
}
type mediaKind int
const (
UNKNOWN mediaKind = iota
PIC
GIF
VIDEO
)
func kind(path string) mediaKind {
ext := filepath.Ext(path)
switch ext {
case ".jpg":
fallthrough
case ".jpeg":
fallthrough
case ".png":
return PIC
case ".gif":
return GIF
case ".mp4":
return VIDEO
}
return UNKNOWN
}
func splitStatus(status string) []string {
characterLimit := CHARACTER_LIMIT
// Twitter has an insane definition of what counts as a character
// ( see https://developer.twitter.com/en/docs/counting-characters )
// - as a crude approximation, anything outside LATIN-1 halfs the limit
for _, ch := range status {
if ch > 0x10FF {
characterLimit = CHARACTER_LIMIT / 2
break
}
}
asRunes := []rune(status)
split := []string{}
for len(asRunes) != 0 {
var limit int
if len(asRunes) <= characterLimit {
limit = len(asRunes)
} else {
tmp := asRunes[0:characterLimit]
lastSpace := strings.LastIndex(string(tmp), " ")
if lastSpace == -1 {
limit = characterLimit
} else {
limit = lastSpace + 1
}
}
split = append(split, string(asRunes[0:limit]))
asRunes = asRunes[limit:]
}
return split
}
func exitIfInvalid(path string) {
log := func(err error) { optLogFatal("exitIfInvalid", err) }
// check existence AND readability
f, err := os.Open(path)
log(err)
defer f.Close()
tmp, err := ioutil.ReadAll(f)
log(err)
if len(tmp) > MAX_BYTES {
log(errors.New("File too big: " + path + " is bigger than maximum of " + strconv.Itoa(MAX_BYTES) + " Bytes"))
}
}
func splitArguments(args []string) data {
if len(args) < 1 {
fmt.Fprintln(os.Stderr, "Usage: drivel status STATUS [FILE1 FILE2 ...]")
os.Exit(-1)
}
d := data{}
d.status = splitStatus(args[0])
for _, arg := range args[1:] {
exitIfInvalid(arg)
switch kind(arg) {
case PIC:
d.pics = append(d.pics, arg)
case GIF:
d.gifs = append(d.gifs, arg)
case VIDEO:
d.videos = append(d.videos, arg)
default:
optLogFatal("splitArguments", errors.New("Unsupported file: "+arg))
}
}
return d
}
func tweet(status string, mediaIDs []ObjectID, previousID ObjectID) ObjectID {
log := func(err error) { optLogFatal("tweet "+status, err) }
request := UpdateStatusRequest(status, mediaIDs, previousID)
body := send(STATUS_ENDPOINT, request)
var sr UpdateStatusResponse
err := json.Unmarshal(body, &sr)
log(err)
if len(sr.Errors) > 0 {
log(sr)
}
fmt.Println("==> Updated status to '" + status + "' with id " + sr.Id_str)
return ObjectID(sr.Id_str)
}
type data struct {
status []string
pics []string
gifs []string
videos []string
}
func (d *data) getStatus(i int) string {
return goutil.StrSliceAt(d.status, i)
}
func (d *data) getGif(i int) string {
return goutil.StrSliceAt(d.gifs, i)
}
func (d *data) getVideo(i int) string {
return goutil.StrSliceAt(d.videos, i)
}
func (d *data) uploadPics(from, to int) []ObjectID {
pics := goutil.StrSlice(d.pics, from, to)
return uploadAll(pics)
}
func (d *data) uploadGif(i int) []ObjectID {
gif := d.getGif(i)
return uploadAll([]string{gif})
}
func (d *data) uploadVideo(i int) []ObjectID {
vid := d.getVideo(i)
return uploadAll([]string{vid})
}
func (d *data) push(previous ObjectID) {
if d == nil {
return
}
empty := false
i, g, v := 0, 0, 0
for !empty {
empty = true
status := d.getStatus(i)
mediaIDs := []ObjectID{}
if status != "" {
empty = false
}
from := i * 4
to := (i + 1) * 4
mediaIDs = d.uploadPics(from, to)
if len(mediaIDs) == 0 {
mediaIDs = d.uploadGif(g)
g++
}
if len(mediaIDs) == 0 {
mediaIDs = d.uploadVideo(v)
v++
}
if len(mediaIDs) > 0 {
empty = false
}
if !empty {
previous = tweet(status, mediaIDs, previous)
i++
}
}
}
func updateStatus(args []string, previous ObjectID, embedTweet ObjectID) {
d := splitArguments(args)
if embedTweet != "" {
tweets := _lookup([]string{string(embedTweet)})
if len(tweets) == 1 {
d.status[0] += " " + tweets[0].URL()
}
}
d.push(previous)
}
func status(args []string) error {
updateStatus(args, "", "")
return nil
}
func reply(args []string) error {
if len(args) < 2 {
fmt.Fprintln(os.Stderr, "Usage: drivel reply TWEET_ID MESSAGE [FILE1 FILE2 ...]")
os.Exit(-1)
}
updateStatus(args[1:], ObjectID(args[0]), "")
return nil
}
func quote(args []string) error {
if len(args) < 2 {
fmt.Println(os.Stderr, "Usage: drivel quote TWEET_ID MESSAGE [FILE1 FILE2 ...]")
os.Exit(-1)
}
updateStatus(args[1:], "", ObjectID(args[0]))
return nil
}
type TwitterTime struct {
time.Time
}
func (twt *TwitterTime) UnmarshalJSON(b []byte) error {
s := strings.Trim(string(b), "\"")
var err error
twt.Time, err = time.Parse(time.RubyDate, s)
return err
}
type Status struct {
Full_text string
Id_str string
Created_at TwitterTime
In_reply_to_screen_name string
In_reply_to_status_id_str string
User StatusUser
Quoted_status *Status
Retweeted_status *Status
Extended_entities Entities
}
func (t Status) equals(t2 Status) bool {
return t.Id_str == t2.Id_str
}
type Entities struct {
Media []Media
}
type Media struct {
Media_url string
}
func (m Status) InReplyTo() string {
if m.In_reply_to_status_id_str != "" {
return m.In_reply_to_screen_name + " (" + m.In_reply_to_status_id_str + ")"
} else {
return ""
}
}
func (m Status) String() string {
if m.Retweeted_status != nil {
return m.User.Screen_name + " retweeted " + m.Retweeted_status.String()
}
s := m.User.Screen_name + " " + "(" + m.Id_str + ")"
if replyTo := m.InReplyTo(); replyTo != "" {
s += " in reply to " + replyTo
}
s += ":\n" + html.UnescapeString(m.Full_text)
allMedia := m.Extended_entities.Media
if len(allMedia) > 0 {
s += "\n\nMedia:"
for _, media := range allMedia {
s += " " + media.Media_url
}
}
if m.Quoted_status != nil {
s += "\n\nQuotes " + m.Quoted_status.String()
}
return s
}
func PrintTweets(tweets []Status, userFilter hashset) {
for _, tweet := range tweets {
if !userFilter.contains(tweet.User.Screen_name) {
fmt.Println(tweet)
fmt.Println("---------")
}
}
}
func (m Status) URL() string {
return "https://twitter.com/" + m.User.Screen_name + "/status/" + m.Id_str
}
type StatusUser struct {
Name string
Screen_name string
}
func _lookup(ids []string) []Status {
log := func(err error) { optLogFatal("lookup "+strings.Join(ids, ","), err) }
body := get(LOOKUP_ENDPOINT + LookupParameters(ids))
var tweets []Status
err := json.Unmarshal(body, &tweets)
log(err)
return tweets
}
func lookup(args []string) error {
if len(args) < 1 {
fmt.Fprintln(os.Stderr, "USAGE: drivel lookup TWEET_ID1 [TWEET_ID2 TWEET_ID3 ...]")
os.Exit(-1)
}
tweets := _lookup(args)
PrintTweets(tweets, nil)
return nil
}
func timeline(endpoint string) []Status {
log := func(err error) { optLogFatal("timeline", err) }
body := get(endpoint)
var tweets []Status
err := json.Unmarshal(body, &tweets)
log(err)
return tweets
}
func mentions(args []string) error {
tweets := timeline(MENTIONS_ENDPOINT)
PrintTweets(tweets, mentionsFilter)
return nil
}
func home(args []string) error {
tweets := timeline(HOME_ENDPOINT)
PrintTweets(tweets, homeFilter)
return nil
}
func UserTimelineParameters(flags userTimelineFlags, screenName string) string {
s := "&screen_name=" + screenName
if flags.withReplies {
return s
} else {
return s + "&exclude_replies=true"
}
}
func userTimeline(flags userTimelineFlags, args []string) error {
tweets := timeline(TIMELINE_ENDPOINT + UserTimelineParameters(flags, args[0]))
PrintTweets(tweets, nil)
return nil
}
type userTimelineFlags struct {
withReplies bool
}
func userTimelineCommand() (goutil.CommandFlagsInit, goutil.CommandFunc) {
f := userTimelineFlags{}
flagsInit := func(s *flag.FlagSet) {
s.BoolVar(&f.withReplies, "with-replies", false, "include replies in timeline")
}
return flagsInit, func(args []string) error { return userTimeline(f, args) }
}
func RetweetParameters(id string) string {
return id + ".json"
}
func retweet(args []string) error {
log := func(err error) { optLogFatal("retweet", err) }
if len(args) != 1 {
fmt.Fprintln(os.Stderr, "USAGE: drivel retweet TWEET_ID")
os.Exit(-1)
}
id := args[0]
tweets := _lookup([]string{id})
if len(tweets) != 1 {
log(errors.New("Could not find tweet " + id))
}
body := send(RETWEET_ENDPOINT+RetweetParameters(id), nil)
var retweet Status
err := json.Unmarshal(body, &retweet)
log(err)
fmt.Println("Retweeted", tweets[0])
return nil
}
func LikeRequest(id string) url.Values {
return map[string][]string{
"id": {id},
"tweet_mode": {"extended"},
}
}
func like(args []string) error {
log := func(err error) { optLogFatal("like", err) }
if len(args) != 1 {
fmt.Fprintln(os.Stderr, "USAGE: drivel like TWEET_ID")
os.Exit(-1)
}
body := send(LIKE_ENDPOINT, LikeRequest(args[0]))
var tweet Status
err := json.Unmarshal(body, &tweet)
log(err)
fmt.Println("Liked", tweet)
return nil
}
func equals(t1 []Status, t2 []Status) bool {
if len(t1) != len(t2) {
return false
}
for i := range t1 {
if !t1[i].equals(t2[i]) {
return false
}
}
return true
}
func UnlikeRequest(id string) url.Values {
return map[string][]string{
"id": {id},
}
}
func unlike(id string) {
log := func(err error) { optLogFatal("unlike", err) }
body := send(DESTROY_LIKE_ENDPOINT, UnlikeRequest(id))
var tweet Status
err := json.Unmarshal(body, &tweet)
log(err)
fmt.Println("Unliked", tweet.Id_str)
}
func DestroyParameters(id string) string {
return id + ".json"
}
func destroyStatus(id string) {
log := func(err error) { optLogFatal("destroy", err) }
body := send(DESTROY_STATUS_ENDPOINT+DestroyParameters(id), nil)
var tweet Status
err := json.Unmarshal(body, &tweet)
log(err)
fmt.Println("Destroyed", tweet.Id_str)
}
func wipeTimeline(likes bool, keepDays int) {
var endpoint string
if likes {
endpoint = LIKES_TIMELINE_ENDPOINT
} else {
endpoint = TIMELINE_ENDPOINT
}
n := 0
now := time.Now()
tweets := timeline(endpoint)
for {
for _, tweet := range tweets {
daysSince := now.Sub(tweet.Created_at.Time).Hours() / 24
if tweet.Created_at != (TwitterTime{}) && daysSince >= float64(keepDays) {
if likes {
unlike(tweet.Id_str)
} else {
destroyStatus(tweet.Id_str)
}
n++
if n >= 200 {
fmt.Println("reached limit of 200")
return
}
}
}
newTweets := timeline(endpoint)
if !equals(newTweets, tweets) {
tweets = newTweets
} else {
return
}
}
}
func wipe(flags wipeFlags) error {
wipeTimeline(true, flags.keepDays)
wipeTimeline(false, flags.keepDays)
return nil
}
type wipeFlags struct {
keepDays int
}
func wipeCommand() (goutil.CommandFlagsInit, goutil.CommandFunc) {
f := wipeFlags{}
flagsInit := func(s *flag.FlagSet) {
s.IntVar(&f.keepDays, "keep-days", WIPE_KEEP_DAYS, "don't wipe the last N days")
}
return flagsInit, func([]string) error { return wipe(f) }
}
type hashset map[string]interface{}
func makeHashset() hashset {
return make(map[string]interface{})
}
func (s hashset) add(member string) {
if s != nil {
s[member] = nil
}
}
func (s hashset) contains(member string) bool {
if s == nil {
return false
}
if _, ok := s[member]; ok {
return true
} else {
return false
}
}
func setFilters(appDir string) {
getHashset := func(s string, err error) hashset {
set := makeHashset()
if err == nil {
for _, id := range strings.Split(s, "\n") {
if strings.TrimSpace(id) != "" {
set.add(id)
}
}
}
return set
}
homeFilter = getHashset(goutil.ReadFile(path.Join(appDir, "FilterHome")))
mentionsFilter = getHashset(goutil.ReadFile(path.Join(appDir, "FilterMentions")))
}
var client *http.Client
var homeFilter hashset
var mentionsFilter hashset
func main() {
client = getClient()
setFilters(appDir())
commands := []goutil.Command{
goutil.NewCommand("status", status, "post a status with message and/or media"),
goutil.NewCommand("home", home, "get your home timeline"),
goutil.NewCommand("mentions", mentions, "get your mention timeline"),
goutil.NewCommandWithFlags("timeline", userTimelineCommand, "get timeline of a specific user"),
goutil.NewCommand("lookup", lookup, "lookup tweets with specific IDs"),
goutil.NewCommand("reply", reply, "reply to a tweet with a specific ID"),
goutil.NewCommand("quote", quote, "quote retweet a tweet with a specific ID"),
goutil.NewCommand("retweet", retweet, "retweet a tweet with a specific ID"),
goutil.NewCommand("like", like, "like a tweet with a specific ID"),
goutil.NewCommandWithFlags("wipe", wipeCommand, "wipe your timeline and likes"),
}
err := goutil.Execute(commands)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}