From 153c0a10ff9e4103714b0aad30a9f5f7571c3c80 Mon Sep 17 00:00:00 2001 From: davigonz Date: Mon, 25 Jun 2018 15:32:00 +0200 Subject: [PATCH] Basic upload with chunks [WIP] --- .../ChunkFromFileChannelRequestEntity.java | 8 +- .../network/ChunkFromFileRequestBody.java | 100 ++++++++++++ .../lib/common/network/FileRequestBody.java | 4 +- .../ChunkedUploadRemoteFileOperation.java | 145 ++++++++++-------- 4 files changed, 184 insertions(+), 73 deletions(-) create mode 100644 src/com/owncloud/android/lib/common/network/ChunkFromFileRequestBody.java diff --git a/src/com/owncloud/android/lib/common/network/ChunkFromFileChannelRequestEntity.java b/src/com/owncloud/android/lib/common/network/ChunkFromFileChannelRequestEntity.java index 8a88723b..f16be1ff 100644 --- a/src/com/owncloud/android/lib/common/network/ChunkFromFileChannelRequestEntity.java +++ b/src/com/owncloud/android/lib/common/network/ChunkFromFileChannelRequestEntity.java @@ -56,7 +56,7 @@ public class ChunkFromFileChannelRequestEntity implements RequestEntity, Progres private final File mFile; private long mOffset; private long mTransferred; - Set mDataTransferListeners = new HashSet(); + Set mDataTransferListeners = new HashSet<>(); private ByteBuffer mBuffer = ByteBuffer.allocate(4096); public ChunkFromFileChannelRequestEntity( @@ -120,8 +120,8 @@ public class ChunkFromFileChannelRequestEntity implements RequestEntity, Progres public void writeRequest(final OutputStream out) throws IOException { - int readCount = 0; - Iterator it = null; + int readCount; + Iterator it; try { mChannel.position(mOffset); @@ -161,7 +161,5 @@ public class ChunkFromFileChannelRequestEntity implements RequestEntity, Progres } catch (FileRequestEntity.WriteException we) { throw we.getWrapped(); } - } - } \ No newline at end of file diff --git a/src/com/owncloud/android/lib/common/network/ChunkFromFileRequestBody.java b/src/com/owncloud/android/lib/common/network/ChunkFromFileRequestBody.java new file mode 100644 index 00000000..2639217b --- /dev/null +++ b/src/com/owncloud/android/lib/common/network/ChunkFromFileRequestBody.java @@ -0,0 +1,100 @@ +/* ownCloud Android Library is available under MIT license + * Copyright (C) 2018 ownCloud GmbH. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +package com.owncloud.android.lib.common.network; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Iterator; + +import okhttp3.MediaType; +import okio.BufferedSink; + +public class ChunkFromFileRequestBody extends FileRequestBody { + + private static final String TAG = ChunkFromFileChannelRequestEntity.class.getSimpleName(); + + //private final File mFile; + private final FileChannel mChannel; + private final long mChunkSize; + private long mOffset; + private long mTransferred; + private ByteBuffer mBuffer = ByteBuffer.allocate(4096); + + public ChunkFromFileRequestBody(File file, MediaType contentType, FileChannel channel, long chunkSize) { + super(file, contentType); + if (channel == null) { + throw new IllegalArgumentException("File may not be null"); + } + if (chunkSize <= 0) { + throw new IllegalArgumentException("Chunk size must be greater than zero"); + } + this.mChannel = channel; + this.mChunkSize = chunkSize; + mOffset = 0; + mTransferred = 0; + } + + + @Override + public void writeTo(BufferedSink sink) { + int readCount; + Iterator it; + + try { + mChannel.position(mOffset); + long size = mFile.length(); + if (size == 0) size = -1; + long maxCount = Math.min(mOffset + mChunkSize, mChannel.size()); + while (mChannel.position() < maxCount) { + readCount = mChannel.read(mBuffer); + sink.write(mBuffer.array(), 0 ,readCount); + mBuffer.clear(); + if (mTransferred < maxCount) { // condition to avoid accumulate progress for repeated chunks + mTransferred += readCount; + } + synchronized (mDataTransferListeners) { + it = mDataTransferListeners.iterator(); + while (it.hasNext()) { + it.next().onTransferProgress(readCount, mTransferred, size, mFile.getAbsolutePath()); + } + } + } + + sink.flush(); + + } catch (IOException io) { +// // any read problem will be handled as if the file is not there +// if (io instanceof FileNotFoundException) { +// throw io; +// } else { +// FileNotFoundException fnf = new FileNotFoundException("Exception reading source file"); +// fnf.initCause(io); +// throw fnf; +// } + } + } +} \ No newline at end of file diff --git a/src/com/owncloud/android/lib/common/network/FileRequestBody.java b/src/com/owncloud/android/lib/common/network/FileRequestBody.java index 9c4b30ac..bec7f041 100644 --- a/src/com/owncloud/android/lib/common/network/FileRequestBody.java +++ b/src/com/owncloud/android/lib/common/network/FileRequestBody.java @@ -43,7 +43,7 @@ import okio.Source; */ public class FileRequestBody extends RequestBody implements ProgressiveDataTransferer { - private File mFile; + protected File mFile; private MediaType mContentType; Set mDataTransferListeners = new HashSet<>(); @@ -71,7 +71,7 @@ public class FileRequestBody extends RequestBody implements ProgressiveDataTrans long transferred = 0; long read; - while ((read = source.read(sink.buffer(), 2048)) != -1) { + while ((read = source.read(sink.buffer(), 4096)) != -1) { transferred += read; sink.flush(); synchronized (mDataTransferListeners) { diff --git a/src/com/owncloud/android/lib/resources/files/ChunkedUploadRemoteFileOperation.java b/src/com/owncloud/android/lib/resources/files/ChunkedUploadRemoteFileOperation.java index 4ba4dedf..df454407 100644 --- a/src/com/owncloud/android/lib/resources/files/ChunkedUploadRemoteFileOperation.java +++ b/src/com/owncloud/android/lib/resources/files/ChunkedUploadRemoteFileOperation.java @@ -1,5 +1,5 @@ /* ownCloud Android Library is available under MIT license - * Copyright (C) 2016 ownCloud GmbH. + * Copyright (C) 2018 ownCloud GmbH. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,14 +26,28 @@ package com.owncloud.android.lib.resources.files; import com.owncloud.android.lib.common.OwnCloudClient; import com.owncloud.android.lib.common.http.HttpUtils; -import com.owncloud.android.lib.common.http.methods.webdav.MkColMethod; -import com.owncloud.android.lib.common.network.WebdavUtils; +import com.owncloud.android.lib.common.http.methods.webdav.PutMethod; +import com.owncloud.android.lib.common.network.ChunkFromFileRequestBody; import com.owncloud.android.lib.common.operations.RemoteOperationResult; +import com.owncloud.android.lib.common.utils.Log_OC; +import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; +import okhttp3.MediaType; + +import static com.owncloud.android.lib.common.http.HttpConstants.IF_MATCH_HEADER; +import static com.owncloud.android.lib.common.http.HttpConstants.OC_TOTAL_LENGTH_HEADER; +import static com.owncloud.android.lib.common.operations.RemoteOperationResult.ResultCode.OK; + +/** + * Remote operation performing the chunked upload of a remote file to the ownCloud server. + * + * @author David A. Velasco + * @author David González Verdugo + */ public class ChunkedUploadRemoteFileOperation extends UploadRemoteFileOperation { private static final int LAST_CHUNK_TIMEOUT = 900000; //15 mins. @@ -46,11 +60,6 @@ public class ChunkedUploadRemoteFileOperation extends UploadRemoteFileOperation private long mTransferId; - public ChunkedUploadRemoteFileOperation(String storagePath, String remotePath, String mimeType, - String fileLastModifTimestamp) { - super(storagePath, remotePath, mimeType, fileLastModifTimestamp); - } - public ChunkedUploadRemoteFileOperation(long transferId, String storagePath, String remotePath, String mimeType, String requiredEtag, String fileLastModifTimestamp) { super(storagePath, remotePath, mimeType, requiredEtag, fileLastModifTimestamp); @@ -65,80 +74,84 @@ public class ChunkedUploadRemoteFileOperation extends UploadRemoteFileOperation FileChannel channel = null; RandomAccessFile raf = null; - //TODO -// try { -// File file = new File(mLocalPath); -// raf = new RandomAccessFile(file, "r"); -// channel = raf.getChannel(); -// mEntity = new ChunkFromFileChannelRequestEntity(channel, mMimeType, CHUNK_SIZE, file); -// synchronized (mDataTransferListeners) { -// ((ProgressiveDataTransferer) mEntity) -// .addDatatransferProgressListeners(mDataTransferListeners); -// } -// -// long offset = 0; -// String uriPrefix = client.getOldWebdavUri() + WebdavUtils.encodePath(mRemotePath) + -// "-chunking-" + Math.abs((new Random()).nextInt(9000) + 1000) + "-"; -// long totalLength = file.length(); -// long chunkCount = (long) Math.ceil((double) totalLength / CHUNK_SIZE); -// String chunkSizeStr = String.valueOf(CHUNK_SIZE); -// String totalLengthStr = String.valueOf(file.length()); -// for (int chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++, offset += CHUNK_SIZE) { -// if (chunkIndex == chunkCount - 1) { -// chunkSizeStr = String.valueOf(CHUNK_SIZE * chunkCount - totalLength); -// } -// if (mPutMethod != null) { -// mPutMethod.releaseConnection(); // let the connection available -// // for other methods -// } -// mPutMethod = new PutMethod(uriPrefix + chunkCount + "-" + chunkIndex); -// if (mRequiredEtag != null && mRequiredEtag.length() > 0) { -// mPutMethod.addRequestHeader(IF_MATCH_HEADER, "\"" + mRequiredEtag + "\""); -// } + try { + File fileToUpload = new File(mLocalPath); + MediaType mediaType = MediaType.parse(mMimeType); + + raf = new RandomAccessFile(fileToUpload, "r"); + channel = raf.getChannel(); + + mFileRequestBody = new ChunkFromFileRequestBody(fileToUpload, mediaType, channel, CHUNK_SIZE); + + synchronized (mDataTransferListeners) { + mFileRequestBody.addDatatransferProgressListeners(mDataTransferListeners); + } + + long offset = 0; + String uriPrefix = client.getNewUploadsWebDavUri() + FileUtils.PATH_SEPARATOR + String.valueOf(mTransferId); + long totalLength = fileToUpload.length(); + long chunkCount = (long) Math.ceil((double) totalLength / CHUNK_SIZE); + String chunkSizeStr = String.valueOf(CHUNK_SIZE); + String totalLengthStr = String.valueOf(fileToUpload.length()); + + for (int chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++, offset += CHUNK_SIZE) { + if (chunkIndex == chunkCount - 1) { + chunkSizeStr = String.valueOf(CHUNK_SIZE * chunkCount - totalLength); + } + + mPutMethod = new PutMethod( + HttpUtils.stringUrlToHttpUrl(uriPrefix + FileUtils.PATH_SEPARATOR + chunkIndex) + ); + + if (mRequiredEtag != null && mRequiredEtag.length() > 0) { + mPutMethod.addRequestHeader(IF_MATCH_HEADER, "\"" + mRequiredEtag + "\""); + } // mPutMethod.addRequestHeader(OC_CHUNKED_HEADER, OC_CHUNKED_HEADER); // mPutMethod.addRequestHeader(OC_CHUNK_SIZE_HEADER, chunkSizeStr); -// mPutMethod.addRequestHeader(OC_TOTAL_LENGTH_HEADER, totalLengthStr); -// -// mPutMethod.addRequestHeader(OC_CHUNK_X_OC_MTIME_HEADER, mFileLastModifTimestamp); -// + mPutMethod.addRequestHeader(OC_TOTAL_LENGTH_HEADER, totalLengthStr); + + mPutMethod.addRequestHeader(OC_CHUNK_X_OC_MTIME_HEADER, mFileLastModifTimestamp); + // ((ChunkFromFileChannelRequestEntity) mEntity).setOffset(offset); // mPutMethod.setRequestEntity(mEntity); // if (mCancellationRequested.get()) { // mPutMethod.abort(); // // next method will throw an exception // } -// + // if (chunkIndex == chunkCount - 1) { // // Added a high timeout to the last chunk due to when the last chunk // // arrives to the server with the last PUT, all chunks get assembled // // within that PHP request, so last one takes longer. // mPutMethod.getParams().setSoTimeout(LAST_CHUNK_TIMEOUT); // } -// -// status = client.executeMethod(mPutMethod); -// -// result = new RemoteOperationResult( -// isSuccess(status), -// mPutMethod -// ); -// -// client.exhaustResponse(mPutMethod.getResponseBodyAsStream()); -// Log_OC.d(TAG, "Upload of " + mLocalPath + " to " + mRemotePath + -// ", chunk index " + chunkIndex + ", count " + chunkCount + -// ", HTTP result status " + status); -// -// if (!isSuccess(status)) -// break; -// } -// -// } finally { -// if (channel != null) -// channel.close(); -// if (raf != null) -// raf.close(); + + mPutMethod.setRequestBody(mFileRequestBody); + + status = client.executeHttpMethod(mPutMethod); + + Log_OC.d(TAG, "Upload of " + mLocalPath + " to " + mRemotePath + + ", chunk index " + chunkIndex + ", count " + chunkCount + + ", HTTP result status " + status); + + if (isSuccess(status)) { + result = new RemoteOperationResult(OK); + } else { + result = new RemoteOperationResult(mPutMethod); + break; + } + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (channel != null) + channel.close(); + if (raf != null) + raf.close(); // if (mPutMethod != null) // mPutMethod.releaseConnection(); // let the connection available for other methods -// } + } return result; } } \ No newline at end of file