large video and large file support

Readme.md

@@ -55,7 +55,7 @@ drivel reply TWEET_ID MESSAGE [FILE1, FILE2, ...]
 ```
 
 
-with any number of files, as long as they are .jpg, .png, .gif or .mp4 and smaller than 5 MB each. On first use, drivel will ask you to go to [https://apps.twitter.com/app/new](https://apps.twitter.com/app/new), register a new app and create an access token. Those values will be stored in HOME/.drivel/ for later use.
+with any number of files, as long as they are .jpg, .png, .gif or .mp4 and smaller than 50 MB each. On first use, drivel will ask you to go to [https://apps.twitter.com/app/new](https://apps.twitter.com/app/new), register a new app and create an access token. Those values will be stored in HOME/.drivel/ for later use.
 
 drivel will automatically split large status messages and multiple files into separate tweets belonging to the same thread.
 

drivel.go

@@ -17,7 +17,8 @@ import (
 )
 
 const (
-	MAX_BYTES         = 5 * 1024 * 1024
+	MAX_BYTES         = 50 * 1024 * 1024
+	CHUNK_SIZE        = 1024 * 1024
 	CHARACTER_LIMIT   = 280
 	UPLOAD_ENDPOINT   = "https://upload.twitter.com/1.1/media/upload.json"
 	STATUS_ENDPOINT   = "https://api.twitter.com/1.1/statuses/update.json"
@@ -27,7 +28,7 @@ const (
 )
 
 func optLogFatal(decorum string, err error) {
-	if err != nil {
+	if err != nil && err.Error() != "" {
 		fmt.Fprintln(os.Stderr, "drivel: "+decorum+": "+err.Error())
 		os.Exit(-1)
 	}
@@ -71,19 +72,19 @@ type InitResponse struct {
 	Media_id_string string
 }
 
-func AppendRequest(mediaID string, mediaData string, segmentIndex int) url.Values {
+func AppendRequest(mediaID ObjectID, mediaData string, segmentIndex int) url.Values {
 	return map[string][]string{
 		"command":       {"APPEND"},
-		"media_id":      {mediaID},
+		"media_id":      {string(mediaID)},
 		"media_data":    {mediaData},
 		"segment_index": {strconv.Itoa(segmentIndex)},
 	}
 }
 
-func FinalizeRequest(mediaID string) url.Values {
+func FinalizeRequest(mediaID ObjectID) url.Values {
 	return map[string][]string{
 		"command":  {"FINALIZE"},
-		"media_id": {mediaID},
+		"media_id": {string(mediaID)},
 	}
 }
 
@@ -96,6 +97,16 @@ type FinalizeResponse struct {
 type ProcessingInfo struct {
 	State            string
 	Check_after_secs int64
+	Progress_percent int64
+	Error            TwitterError
+}
+
+type PollStatusResponse struct {
+	Processing_info ProcessingInfo
+}
+
+func PollStatusParameters(mediaID ObjectID) string {
+	return "?command=STATUS&media_id=" + string(mediaID)
 }
 
 func UpdateStatusRequest(status string, mediaIDs []ObjectID, previousStatusID ObjectID) url.Values {
@@ -147,7 +158,7 @@ func send(client *http.Client, url string, vals url.Values) []byte {
 func _send(client *http.Client, url string, vals url.Values, usePost bool) []byte {
 	log := func(err error) {
 		v, _ := json.Marshal(vals)
-		optLogFatal("send "+url+" "+string(v), err)
+		optLogFatal("get/post "+url+" "+string(v), err)
 	}
 	var resp *http.Response
 	var err error
@@ -168,37 +179,114 @@ func _send(client *http.Client, url string, vals url.Values, usePost bool) []byt
 	return body
 }
 
-func uploadFile(client *http.Client, file string) ObjectID {
+func initFileUpload(client *http.Client, file string, mediaData []byte) ObjectID {
-	log := func(err error) { optLogFatal("uploadFile "+file, err) }
+	log := func(err error) { optLogFatal("initFileUpload "+file, err) }
-	media, err := ioutil.ReadFile(file)
+	initRequest := InitRequest(getMimetype(file), len(mediaData))
-	log(err)
-	initRequest := InitRequest(getMimetype(file), len(media))
 	body := send(client, UPLOAD_ENDPOINT, initRequest)
 	var initResponse InitResponse
-	err = json.Unmarshal(body, &initResponse)
+	err := json.Unmarshal(body, &initResponse)
 	log(err)
-	if len(initResponse.Errors) == 0 {
+	log(initResponse)
-		mediaId := initResponse.Media_id_string
+	return ObjectID(initResponse.Media_id_string)
-		appRequest := AppendRequest(mediaId, base64.RawURLEncoding.EncodeToString(media), 0)
+}
-		body := send(client, UPLOAD_ENDPOINT, appRequest)
+
-		if string(body) == "" {
+func appendFileChunks(client *http.Client, file string, media string, mediaId ObjectID) {
-			body := send(client, UPLOAD_ENDPOINT, FinalizeRequest(mediaId))
+	log := func(err error) { optLogFatal("appendFileChunks", err) }
-			var finalizeResponse FinalizeResponse
+	info := func(v ...interface{}) {
-			json.Unmarshal(body, &finalizeResponse)
+		if len(media) > CHUNK_SIZE {
-			if id := ObjectID(finalizeResponse.Media_id_string); id != "" {
+			fmt.Println(v...)
-				fmt.Println("==> Uploaded " + file + " with id " + string(id))
-				procInfo := finalizeResponse.Processing_info
-				if procInfo != (ProcessingInfo{}) && procInfo.Check_after_secs != 0 {
-					wait := procInfo.Check_after_secs * 2
-					fmt.Println("Need to wait", wait, "seconds")
-					time.Sleep(time.Duration(wait) * time.Second)
-				}
-				return id
-			}
 		}
 	}
-	log(errors.New("Could not upload file " + file))
+	info("chunk upload", file)
-	return ""
+	info("total", len(media))
+	for i := 0; i*CHUNK_SIZE < len(media); i = i + 1 {
+		start := i * CHUNK_SIZE
+		end := (i + 1) * CHUNK_SIZE
+		if end > len(media) {
+			end = len(media)
+		}
+		info("segment", i, "start", start, "end", end)
+		appended := false
+		var body []byte
+		for try := 0; try < 3 && !appended; try++ {
+			appRequest := AppendRequest(mediaId, media[start:end], i)
+			body = send(client, UPLOAD_ENDPOINT, appRequest)
+			if string(body) == "" {
+				appended = true
+			}
+		}
+		if !appended {
+			log(errors.New(string(body)))
+		}
+	}
+}
+
+func finalizeFileUpload(client *http.Client, file string, mediaId ObjectID) int64 {
+	log := func(err error) { optLogFatal("finalizeFileUpload", err) }
+	body := send(client, UPLOAD_ENDPOINT, FinalizeRequest(mediaId))
+	var finalizeResponse FinalizeResponse
+	err := json.Unmarshal(body, &finalizeResponse)
+	log(err)
+	log(errors.New(finalizeResponse.Error))
+	if id := ObjectID(finalizeResponse.Media_id_string); id != "" {
+		fmt.Println("==> Uploaded " + file + " with id " + string(id))
+		procInfo := finalizeResponse.Processing_info
+		return procInfo.Check_after_secs
+	} else {
+		log(errors.New("Could not finalize " + string(mediaId)))
+		return 0
+	}
+}
+
+func wait(seconds int64) {
+	fmt.Println("Waiting", seconds, "seconds")
+	time.Sleep(time.Duration(seconds) * time.Second)
+}
+
+func pollStatus(client *http.Client, mediaId ObjectID) {
+	log := func(err error) { optLogFatal("pollStatus "+string(mediaId), err) }
+	succeeded := false
+	var error TwitterError
+	for try := 0; try < 6 && !succeeded; try = try + 1 {
+		body := get(client, UPLOAD_ENDPOINT+PollStatusParameters(mediaId))
+		var response PollStatusResponse
+		err := json.Unmarshal(body, &response)
+		log(err)
+		procInfo := response.Processing_info
+		state := procInfo.State
+		error = procInfo.Error
+		if state == "succeeded" {
+			succeeded = true
+			break
+		} else if state == "failed" {
+			break
+		} else {
+			fmt.Println("Processing progress: ", procInfo.Progress_percent, "%")
+			seconds := procInfo.Check_after_secs
+			if seconds > 10 {
+				seconds = 10
+			}
+			wait(seconds)
+		}
+	}
+	if !succeeded {
+		log(errors.New("File upload failed " + error.Message))
+	}
+}
+
+func uploadFile(client *http.Client, file string) ObjectID {
+	log := func(err error) { optLogFatal("uploadFile "+file, err) }
+	tmpMedia, err := ioutil.ReadFile(file)
+	log(err)
+	media := base64.RawURLEncoding.EncodeToString(tmpMedia)
+	mediaId := initFileUpload(client, file, tmpMedia)
+	appendFileChunks(client, file, media, mediaId)
+	seconds := finalizeFileUpload(client, file, mediaId)
+	if seconds > 0 {
+		wait(seconds)
+		pollStatus(client, mediaId)
+	}
+	return mediaId
 }
 
 func uploadAll(client *http.Client, files []string) []ObjectID {