From f3297655eff67e7752a8af77243c0f9a3678dbcd Mon Sep 17 00:00:00 2001 From: gutmet Date: Fri, 18 Sep 2020 23:22:42 +0200 Subject: [PATCH] large video and large file support --- Readme.md | 2 +- drivel.go | 154 ++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 122 insertions(+), 34 deletions(-) diff --git a/Readme.md b/Readme.md index 5539794..22098ba 100644 --- a/Readme.md +++ b/Readme.md @@ -55,7 +55,7 @@ drivel reply TWEET_ID MESSAGE [FILE1, FILE2, ...] ``` -with any number of files, as long as they are .jpg, .png, .gif or .mp4 and smaller than 5 MB each. On first use, drivel will ask you to go to [https://apps.twitter.com/app/new](https://apps.twitter.com/app/new), register a new app and create an access token. Those values will be stored in HOME/.drivel/ for later use. +with any number of files, as long as they are .jpg, .png, .gif or .mp4 and smaller than 50 MB each. On first use, drivel will ask you to go to [https://apps.twitter.com/app/new](https://apps.twitter.com/app/new), register a new app and create an access token. Those values will be stored in HOME/.drivel/ for later use. drivel will automatically split large status messages and multiple files into separate tweets belonging to the same thread. diff --git a/drivel.go b/drivel.go index e8eb5b2..49cad93 100644 --- a/drivel.go +++ b/drivel.go @@ -17,7 +17,8 @@ import ( ) const ( - MAX_BYTES = 5 * 1024 * 1024 + MAX_BYTES = 50 * 1024 * 1024 + CHUNK_SIZE = 1024 * 1024 CHARACTER_LIMIT = 280 UPLOAD_ENDPOINT = "https://upload.twitter.com/1.1/media/upload.json" STATUS_ENDPOINT = "https://api.twitter.com/1.1/statuses/update.json" @@ -27,7 +28,7 @@ const ( ) func optLogFatal(decorum string, err error) { - if err != nil { + if err != nil && err.Error() != "" { fmt.Fprintln(os.Stderr, "drivel: "+decorum+": "+err.Error()) os.Exit(-1) } @@ -71,19 +72,19 @@ type InitResponse struct { Media_id_string string } -func AppendRequest(mediaID string, mediaData string, segmentIndex int) url.Values { +func AppendRequest(mediaID ObjectID, mediaData string, segmentIndex int) url.Values { return map[string][]string{ "command": {"APPEND"}, - "media_id": {mediaID}, + "media_id": {string(mediaID)}, "media_data": {mediaData}, "segment_index": {strconv.Itoa(segmentIndex)}, } } -func FinalizeRequest(mediaID string) url.Values { +func FinalizeRequest(mediaID ObjectID) url.Values { return map[string][]string{ "command": {"FINALIZE"}, - "media_id": {mediaID}, + "media_id": {string(mediaID)}, } } @@ -96,6 +97,16 @@ type FinalizeResponse struct { type ProcessingInfo struct { State string Check_after_secs int64 + Progress_percent int64 + Error TwitterError +} + +type PollStatusResponse struct { + Processing_info ProcessingInfo +} + +func PollStatusParameters(mediaID ObjectID) string { + return "?command=STATUS&media_id=" + string(mediaID) } func UpdateStatusRequest(status string, mediaIDs []ObjectID, previousStatusID ObjectID) url.Values { @@ -147,7 +158,7 @@ func send(client *http.Client, url string, vals url.Values) []byte { func _send(client *http.Client, url string, vals url.Values, usePost bool) []byte { log := func(err error) { v, _ := json.Marshal(vals) - optLogFatal("send "+url+" "+string(v), err) + optLogFatal("get/post "+url+" "+string(v), err) } var resp *http.Response var err error @@ -168,37 +179,114 @@ func _send(client *http.Client, url string, vals url.Values, usePost bool) []byt return body } -func uploadFile(client *http.Client, file string) ObjectID { - log := func(err error) { optLogFatal("uploadFile "+file, err) } - media, err := ioutil.ReadFile(file) - log(err) - initRequest := InitRequest(getMimetype(file), len(media)) +func initFileUpload(client *http.Client, file string, mediaData []byte) ObjectID { + log := func(err error) { optLogFatal("initFileUpload "+file, err) } + initRequest := InitRequest(getMimetype(file), len(mediaData)) body := send(client, UPLOAD_ENDPOINT, initRequest) var initResponse InitResponse - err = json.Unmarshal(body, &initResponse) + err := json.Unmarshal(body, &initResponse) log(err) - if len(initResponse.Errors) == 0 { - mediaId := initResponse.Media_id_string - appRequest := AppendRequest(mediaId, base64.RawURLEncoding.EncodeToString(media), 0) - body := send(client, UPLOAD_ENDPOINT, appRequest) - if string(body) == "" { - body := send(client, UPLOAD_ENDPOINT, FinalizeRequest(mediaId)) - var finalizeResponse FinalizeResponse - json.Unmarshal(body, &finalizeResponse) - if id := ObjectID(finalizeResponse.Media_id_string); id != "" { - fmt.Println("==> Uploaded " + file + " with id " + string(id)) - procInfo := finalizeResponse.Processing_info - if procInfo != (ProcessingInfo{}) && procInfo.Check_after_secs != 0 { - wait := procInfo.Check_after_secs * 2 - fmt.Println("Need to wait", wait, "seconds") - time.Sleep(time.Duration(wait) * time.Second) - } - return id - } + log(initResponse) + return ObjectID(initResponse.Media_id_string) +} + +func appendFileChunks(client *http.Client, file string, media string, mediaId ObjectID) { + log := func(err error) { optLogFatal("appendFileChunks", err) } + info := func(v ...interface{}) { + if len(media) > CHUNK_SIZE { + fmt.Println(v...) } } - log(errors.New("Could not upload file " + file)) - return "" + info("chunk upload", file) + info("total", len(media)) + for i := 0; i*CHUNK_SIZE < len(media); i = i + 1 { + start := i * CHUNK_SIZE + end := (i + 1) * CHUNK_SIZE + if end > len(media) { + end = len(media) + } + info("segment", i, "start", start, "end", end) + appended := false + var body []byte + for try := 0; try < 3 && !appended; try++ { + appRequest := AppendRequest(mediaId, media[start:end], i) + body = send(client, UPLOAD_ENDPOINT, appRequest) + if string(body) == "" { + appended = true + } + } + if !appended { + log(errors.New(string(body))) + } + } +} + +func finalizeFileUpload(client *http.Client, file string, mediaId ObjectID) int64 { + log := func(err error) { optLogFatal("finalizeFileUpload", err) } + body := send(client, UPLOAD_ENDPOINT, FinalizeRequest(mediaId)) + var finalizeResponse FinalizeResponse + err := json.Unmarshal(body, &finalizeResponse) + log(err) + log(errors.New(finalizeResponse.Error)) + if id := ObjectID(finalizeResponse.Media_id_string); id != "" { + fmt.Println("==> Uploaded " + file + " with id " + string(id)) + procInfo := finalizeResponse.Processing_info + return procInfo.Check_after_secs + } else { + log(errors.New("Could not finalize " + string(mediaId))) + return 0 + } +} + +func wait(seconds int64) { + fmt.Println("Waiting", seconds, "seconds") + time.Sleep(time.Duration(seconds) * time.Second) +} + +func pollStatus(client *http.Client, mediaId ObjectID) { + log := func(err error) { optLogFatal("pollStatus "+string(mediaId), err) } + succeeded := false + var error TwitterError + for try := 0; try < 6 && !succeeded; try = try + 1 { + body := get(client, UPLOAD_ENDPOINT+PollStatusParameters(mediaId)) + var response PollStatusResponse + err := json.Unmarshal(body, &response) + log(err) + procInfo := response.Processing_info + state := procInfo.State + error = procInfo.Error + if state == "succeeded" { + succeeded = true + break + } else if state == "failed" { + break + } else { + fmt.Println("Processing progress: ", procInfo.Progress_percent, "%") + seconds := procInfo.Check_after_secs + if seconds > 10 { + seconds = 10 + } + wait(seconds) + } + } + if !succeeded { + log(errors.New("File upload failed " + error.Message)) + } +} + +func uploadFile(client *http.Client, file string) ObjectID { + log := func(err error) { optLogFatal("uploadFile "+file, err) } + tmpMedia, err := ioutil.ReadFile(file) + log(err) + media := base64.RawURLEncoding.EncodeToString(tmpMedia) + mediaId := initFileUpload(client, file, tmpMedia) + appendFileChunks(client, file, media, mediaId) + seconds := finalizeFileUpload(client, file, mediaId) + if seconds > 0 { + wait(seconds) + pollStatus(client, mediaId) + } + return mediaId } func uploadAll(client *http.Client, files []string) []ObjectID {