Tilman
509 lines
16 KiB
Java
509 lines
16 KiB
Java
/* Copyright (c) 2008 Google Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
|
|
package sample.docs;
|
|
|
|
import com.google.common.collect.Lists;
|
|
import com.google.common.collect.Maps;
|
|
import com.google.gdata.client.media.ResumableGDataFileUploader;
|
|
import sample.util.SimpleCommandLineParser;
|
|
import com.google.gdata.data.Link;
|
|
import com.google.gdata.data.docs.DocumentListEntry;
|
|
import com.google.gdata.data.docs.DocumentListFeed;
|
|
import com.google.gdata.data.media.MediaFileSource;
|
|
import com.google.gdata.util.ServiceException;
|
|
import com.google.gdata.client.uploader.FileUploadData;
|
|
import com.google.gdata.client.uploader.ProgressListener;
|
|
import com.google.gdata.client.uploader.ResumableHttpFileUploader;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.io.PrintStream;
|
|
import java.net.URL;
|
|
import java.util.Collection;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.concurrent.ExecutorService;
|
|
import java.util.concurrent.Executors;
|
|
|
|
/**
|
|
* A console aplication to demonstrate interaction with Google Docs API to
|
|
* upload/update large media files using Resumable Upload protocol.
|
|
*
|
|
*
|
|
*/
|
|
public class DocumentResumableUploadDemo {
|
|
|
|
/** Default document list feed url. */
|
|
private static final String DEFAULT_DOCLIST_FEED_URL =
|
|
"https://docs.google.com/feeds/default/private/full";
|
|
|
|
/** Default create-media-url for uploading documents */
|
|
private static final String DEFAULT_RESUMABLE_UPLOAD_URL =
|
|
"https://docs.google.com/feeds/upload/create-session/default/private/full";
|
|
|
|
/** Maximum number of concurrent uploads */
|
|
private static final int MAX_CONCURRENT_UPLOADS = 10;
|
|
|
|
/** Time interval at which upload task will notify about the progress */
|
|
private static final int PROGRESS_UPDATE_INTERVAL = 1000;
|
|
|
|
/** Max size for each upload chunk */
|
|
private static final int DEFAULT_CHUNK_SIZE = 10000000;
|
|
|
|
/**
|
|
* Welcome message, introducing the program.
|
|
*/
|
|
private static final String[] WELCOME_MESSAGE = {
|
|
"", "This is a demo of the resumable upload feature for Docs GData API",
|
|
"Using this interface, you can upload/update large documents.", ""};
|
|
|
|
private static final String APPLICATION_NAME = "JavaGDataClientSampleAppV3.0";
|
|
|
|
private static final String[] USAGE_MESSAGE = {
|
|
"Usage: java DocumentResumableUploadDemo.jar --username <user> --password <pass>",
|
|
};
|
|
|
|
private static final String[] COMMAND_HELP_MESSAGE = {
|
|
"Commands:",
|
|
" list [object_type] "
|
|
+ " [[list objects]]",
|
|
" upload <file_path1:title1> ... <file_pathN:titleN> "
|
|
+ "[<chunk_size_in_byes>] [[uploads set of files]]",
|
|
" update <ducument_id> <updated_file_path> [<chunk_size_in_bytes>] "
|
|
+ " [[updates content of an object]]",
|
|
};
|
|
|
|
/** Instance of {@link DocumentList} */
|
|
private final DocumentList docs;
|
|
|
|
/** Steam to print status messages to. */
|
|
PrintStream output;
|
|
|
|
/**
|
|
* Constructor.
|
|
*
|
|
* @param docs {@link DocumentList} for interface to DocList API service.
|
|
* @param out printstream to output status messages to.
|
|
*/
|
|
DocumentResumableUploadDemo(DocumentList docs, PrintStream out) {
|
|
this.docs = docs;
|
|
this.output = out;
|
|
}
|
|
|
|
/**
|
|
* Prints out a message.
|
|
*
|
|
* @param msg the message to be printed.
|
|
*/
|
|
private static void printMessage(String[] msg) {
|
|
for (String s : msg) {
|
|
System.out.println(s);
|
|
}
|
|
}
|
|
|
|
private String[] parseCommand(String command) {
|
|
return command.trim().split(" ");
|
|
}
|
|
|
|
/**
|
|
* Prints out the specified document entry.
|
|
*
|
|
* @param doc the document entry to print.
|
|
*/
|
|
public void printDocumentEntry(DocumentListEntry doc) {
|
|
StringBuffer outputBuffer = new StringBuffer();
|
|
|
|
outputBuffer.append(" -- " + doc.getTitle().getPlainText() + " ");
|
|
if (!doc.getParentLinks().isEmpty()) {
|
|
for (Link link : doc.getParentLinks()) {
|
|
outputBuffer.append("[" + link.getTitle() + "] ");
|
|
}
|
|
}
|
|
outputBuffer.append(doc.getResourceId());
|
|
|
|
output.println(outputBuffer);
|
|
}
|
|
|
|
/**
|
|
* Uploads given collection of files. The call blocks until all uploads are
|
|
* done.
|
|
*
|
|
* @param url create-session url for initiating resumable uploads for
|
|
* documents API.
|
|
* @param files list of absolute filepaths to files to upload.
|
|
* @param chunkSize max size of each upload chunk.
|
|
*/
|
|
public Collection<DocumentListEntry> uploadFiles(String url,
|
|
List<String> files, int chunkSize)
|
|
throws IOException, ServiceException, InterruptedException {
|
|
// Create a listener
|
|
FileUploadProgressListener listener = new FileUploadProgressListener();
|
|
// Pool for handling concurrent upload tasks
|
|
ExecutorService executor =
|
|
Executors.newFixedThreadPool(MAX_CONCURRENT_UPLOADS);
|
|
// Create {@link ResumableGDataFileUploader} for each file to upload
|
|
List<ResumableGDataFileUploader> uploaders = Lists.newArrayList();
|
|
for (String fileName : files) {
|
|
MediaFileSource mediaFile = getMediaFileSource(fileName);
|
|
ResumableGDataFileUploader uploader =
|
|
new ResumableGDataFileUploader.Builder(
|
|
docs.service, new URL(url), mediaFile, null /*empty meatadata*/)
|
|
.title(mediaFile.getName())
|
|
.chunkSize(chunkSize).executor(executor)
|
|
.trackProgress(listener, PROGRESS_UPDATE_INTERVAL)
|
|
.build();
|
|
uploaders.add(uploader);
|
|
}
|
|
// attach the listener to list of uploaders
|
|
listener.listenTo(uploaders);
|
|
|
|
// Start the upload
|
|
for (ResumableGDataFileUploader uploader : uploaders) {
|
|
uploader.start();
|
|
}
|
|
|
|
// wait for uploads to complete
|
|
while (!listener.isDone()) {
|
|
try {
|
|
Thread.sleep(100);
|
|
} catch (InterruptedException ie) {
|
|
listener.printResults();
|
|
throw ie; // rethrow
|
|
}
|
|
}
|
|
|
|
// print upload results
|
|
listener.printResults();
|
|
|
|
// return list of uploaded entries
|
|
return listener.getUploaded();
|
|
}
|
|
|
|
private MediaFileSource getMediaFileSource(String fileName) {
|
|
File file = new File(fileName);
|
|
MediaFileSource mediaFile = new MediaFileSource(file,
|
|
DocumentListEntry.MediaType.fromFileName(file.getName())
|
|
.getMimeType());
|
|
return mediaFile;
|
|
}
|
|
|
|
/**
|
|
* Execute 'list' command.
|
|
*/
|
|
private void executeList(String[] args) throws IOException,
|
|
ServiceException, DocumentListException {
|
|
DocumentListFeed feed = null;
|
|
String msg = "";
|
|
|
|
switch (args.length) {
|
|
case 1:
|
|
msg = "List of docs: ";
|
|
feed = docs.getDocsListFeed("all");
|
|
break;
|
|
case 2:
|
|
msg = "List of all " + args[1] + ": ";
|
|
feed = docs.getDocsListFeed(args[1]);
|
|
break;
|
|
case 3:
|
|
if (args[1].equals("folder")) {
|
|
msg = "Contents of folder_id '" + args[2] + "': ";
|
|
feed = docs.getFolderDocsListFeed(args[2]);
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (feed != null) {
|
|
output.println(msg);
|
|
for (DocumentListEntry entry : feed.getEntries()) {
|
|
printDocumentEntry(entry);
|
|
}
|
|
} else {
|
|
printMessage(COMMAND_HELP_MESSAGE);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Execute 'upload' command.
|
|
*/
|
|
private void executeUpload(String[] args)
|
|
throws IOException, ServiceException, InterruptedException {
|
|
if (args.length > 1) {
|
|
int chunkSize = DEFAULT_CHUNK_SIZE;
|
|
List<String> files = Lists.newArrayList();
|
|
for (int index = 1; index < args.length; index++) {
|
|
String arg = args[index];
|
|
if (index < args.length - 1) {
|
|
files.add(arg);
|
|
continue;
|
|
}
|
|
// Last argument can be a file or chunk size
|
|
try {
|
|
chunkSize = Integer.parseInt(arg);
|
|
} catch (NumberFormatException nfe) {
|
|
files.add(arg);
|
|
}
|
|
}
|
|
uploadFiles(DEFAULT_RESUMABLE_UPLOAD_URL, files, chunkSize);
|
|
output.println("Finished upload");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Execute 'update' command.
|
|
*/
|
|
private void executeUpdate(String[] args)
|
|
throws IOException, ServiceException, InterruptedException {
|
|
String docIdToUpdate = args[1];
|
|
String filePath = args[2];
|
|
|
|
// retrieve latest entry
|
|
DocumentListEntry currentEntry = docs.service.getEntry(
|
|
new URL(DEFAULT_DOCLIST_FEED_URL + "/" + docIdToUpdate),
|
|
DocumentListEntry.class);
|
|
|
|
MediaFileSource mediaFile = getMediaFileSource(filePath);
|
|
ResumableGDataFileUploader uploader =
|
|
new ResumableGDataFileUploader
|
|
.Builder(docs.service, mediaFile, currentEntry)
|
|
.title(mediaFile.getName())
|
|
.requestType(
|
|
ResumableGDataFileUploader.RequestType.UPDATE_MEDIA_ONLY)
|
|
.build();
|
|
|
|
uploader.start();
|
|
|
|
// wait for upload to complete
|
|
while (!uploader.isDone()) {
|
|
try {
|
|
Thread.sleep(100);
|
|
} catch (InterruptedException ie) {
|
|
output.println("Media update interrupted at: "
|
|
+ String.format("%3.0f", uploader.getProgress() * 100) + "%");
|
|
throw ie; // rethrow
|
|
}
|
|
}
|
|
DocumentListEntry updatedEntry =
|
|
uploader.getResponse(DocumentListEntry.class);
|
|
|
|
output.println("Finished update");
|
|
}
|
|
|
|
private boolean executeCommand(BufferedReader reader) throws IOException,
|
|
ServiceException, InterruptedException {
|
|
|
|
output.println("Enter a command");
|
|
|
|
try {
|
|
String command = reader.readLine();
|
|
if (command == null) {
|
|
return false;
|
|
}
|
|
|
|
String[] args = parseCommand(command);
|
|
String name = args[0];
|
|
|
|
if (name.equals("list")) {
|
|
executeList(args);
|
|
} else if (name.equals("upload")) {
|
|
executeUpload(args);
|
|
} else if (name.equals("update")) {
|
|
executeUpdate(args);
|
|
} else if (name.startsWith("q") || name.startsWith("exit")) {
|
|
return false;
|
|
} else if (name.equals("help")) {
|
|
printMessage(COMMAND_HELP_MESSAGE);
|
|
} else {
|
|
output.println("Unknown command. Type 'help' for list of commands");
|
|
}
|
|
} catch (DocumentListException e) {
|
|
e.printStackTrace();
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void run() throws IOException, ServiceException, InterruptedException {
|
|
printMessage(WELCOME_MESSAGE);
|
|
printMessage(COMMAND_HELP_MESSAGE);
|
|
|
|
BufferedReader reader = new BufferedReader(
|
|
new InputStreamReader(System.in));
|
|
|
|
while (executeCommand(reader)) {
|
|
}
|
|
}
|
|
|
|
public static void main(String[] args) throws DocumentListException,
|
|
IOException, ServiceException, InterruptedException {
|
|
|
|
SimpleCommandLineParser parser = new SimpleCommandLineParser(args);
|
|
String user = parser.getValue("username", "user", "u");
|
|
String password = parser.getValue("password", "pass", "p");
|
|
boolean help = parser.containsKey("help", "h");
|
|
|
|
if (help || (user == null || password == null)) {
|
|
printMessage(USAGE_MESSAGE);
|
|
System.exit(1);
|
|
}
|
|
|
|
// authenticate
|
|
DocumentList docs = new DocumentList(APPLICATION_NAME);
|
|
docs.login(user, password);
|
|
|
|
DocumentResumableUploadDemo demo = new DocumentResumableUploadDemo(
|
|
docs, System.out);
|
|
demo.run();
|
|
System.exit(1);
|
|
}
|
|
|
|
/**
|
|
* A {@link ProgressListener} implementation to track upload progress.
|
|
* The listener can track multiple uploads at the same time.
|
|
* Use {@link #isDone} to check if all uploads are completed and
|
|
* use {@link #getUploaded} to access results of successful uploads.
|
|
*/
|
|
private class FileUploadProgressListener implements ProgressListener {
|
|
|
|
private Collection<ResumableGDataFileUploader> trackedUploaders
|
|
= Lists.newArrayList();
|
|
private int pendingRequests;
|
|
Map<String, DocumentListEntry> uploaded = Maps.newHashMap();
|
|
Map<String, String> failed = Maps.newHashMap();
|
|
|
|
boolean processed;
|
|
|
|
public FileUploadProgressListener() {
|
|
this.pendingRequests = 0;
|
|
}
|
|
|
|
public void listenTo(Collection<ResumableGDataFileUploader> uploaders) {
|
|
this.trackedUploaders.addAll(uploaders);
|
|
this.pendingRequests = trackedUploaders.size();
|
|
}
|
|
|
|
public synchronized void progressChanged(ResumableHttpFileUploader uploader)
|
|
{
|
|
String fileId = ((FileUploadData) uploader.getData()).getFileName();
|
|
switch(uploader.getUploadState()) {
|
|
case COMPLETE:
|
|
case CLIENT_ERROR:
|
|
pendingRequests -= 1;
|
|
output.println(fileId + ": Completed");
|
|
break;
|
|
case IN_PROGRESS:
|
|
output.println(fileId + ":"
|
|
+ String.format("%3.0f", uploader.getProgress() * 100) + "%");
|
|
break;
|
|
case NOT_STARTED:
|
|
output.println(fileId + ":" + "Not Started");
|
|
break;
|
|
}
|
|
}
|
|
|
|
public synchronized boolean isDone() {
|
|
// not done if there are any pending requests.
|
|
if (pendingRequests > 0) {
|
|
return false;
|
|
}
|
|
// if all responses are processed., nothing to do.
|
|
if (processed) {
|
|
return true;
|
|
}
|
|
// check if all response streams are available.
|
|
for (ResumableGDataFileUploader uploader : trackedUploaders) {
|
|
if (!uploader.isDone()) {
|
|
return false;
|
|
}
|
|
}
|
|
// process all responses
|
|
for (ResumableGDataFileUploader uploader : trackedUploaders) {
|
|
String fileId = ((FileUploadData) uploader.getData()).getFileName();
|
|
switch(uploader.getUploadState()) {
|
|
case COMPLETE:
|
|
try {
|
|
DocumentListEntry entry =
|
|
uploader.getResponse(DocumentListEntry.class);
|
|
uploaded.put(fileId, entry);
|
|
} catch (IOException e) {
|
|
failed.put(fileId, "Upload completed, but unexpected error "
|
|
+ "reading server response");
|
|
} catch (ServiceException e) {
|
|
failed.put(fileId,
|
|
"Upload completed, but failed to parse server response");
|
|
}
|
|
break;
|
|
case CLIENT_ERROR:
|
|
failed.put(fileId, "Failed at " + uploader.getProgress());
|
|
break;
|
|
}
|
|
}
|
|
processed = true;
|
|
output.println("All requests done");
|
|
return true;
|
|
}
|
|
|
|
public synchronized Collection<DocumentListEntry> getUploaded() {
|
|
if (!isDone()) {
|
|
return null;
|
|
}
|
|
return uploaded.values();
|
|
}
|
|
|
|
public synchronized void printResults() {
|
|
if (!isDone()) {
|
|
return;
|
|
}
|
|
output.println("Result: " + uploaded.size() + ", " + failed.size());
|
|
if (uploaded.size() > 0) {
|
|
output.println(" Successfully Uploaded:");
|
|
for (Map.Entry<String, DocumentListEntry> entry : uploaded.entrySet()) {
|
|
printDocumentEntry(entry.getValue());
|
|
}
|
|
}
|
|
if (failed.size() > 0) {
|
|
output.println(" Failed to upload:");
|
|
for (Map.Entry entry : failed.entrySet()) {
|
|
output.println(" " + entry.getKey() + ":" + entry.getValue());
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Prints out the specified document entry.
|
|
*
|
|
* @param doc the document entry to print.
|
|
*/
|
|
public void printDocumentEntry(DocumentListEntry doc) {
|
|
StringBuffer buffer = new StringBuffer();
|
|
|
|
buffer.append(" -- " + doc.getTitle().getPlainText() + " ");
|
|
if (!doc.getParentLinks().isEmpty()) {
|
|
for (Link link : doc.getParentLinks()) {
|
|
buffer.append("[" + link.getTitle() + "] ");
|
|
}
|
|
}
|
|
buffer.append(doc.getResourceId());
|
|
|
|
output.println(buffer);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
}
|