3

My project has been created by GAE Plugin for Eclipse (without Maven) and i'm goint to post my code composed by:

home.jsp

<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
    <head>
    <title>Upload Test</title>
    </head>
    <body>
        <form action="/upload" method="post" name="putFile" id="putFile"
                enctype="multipart/form-data">
                <input type="file" name="myFile" id="fileName">
                <input type="submit" value="Upload">
        </form> 
    </body>
    </html>

UploadServlet.java:

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.nio.channels.Channels;
import java.util.Enumeration;
import java.util.logging.Logger;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.servlet.ServletFileUpload;

import com.google.appengine.tools.cloudstorage.GcsFileOptions;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsOutputChannel;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.appengine.tools.cloudstorage.RetryParams;

public class UploadServlet extends HttpServlet {

    private static final Logger log = Logger.getLogger(UploadServlet.class.getName());

    private final GcsService gcsService = GcsServiceFactory.createGcsService(new RetryParams.Builder()
    .initialRetryDelayMillis(10)
    .retryMaxAttempts(10)
    .totalRetryPeriodMillis(15000)
    .build());

    private String bucketName = "myBucketNameOnGoogleCloudStorage";

    /**Used below to determine the size of chucks to read in. Should be > 1kb and < 10MB */
      private static final int BUFFER_SIZE = 2 * 1024 * 1024;

    @SuppressWarnings("unchecked")
    @Override
    public void doPost(HttpServletRequest req, HttpServletResponse res)
            throws ServletException, IOException {

        String sctype = null, sfieldname, sname = null;
        ServletFileUpload upload;
        FileItemIterator iterator;
        FileItemStream item;
        InputStream stream = null;
        try {
            upload = new ServletFileUpload();
            res.setContentType("text/plain");

            iterator = upload.getItemIterator(req);
            while (iterator.hasNext()) {
                item = iterator.next();
                stream = item.openStream();

                if (item.isFormField()) {
                    log.warning("Got a form field: " + item.getFieldName());
                } else {
                    log.warning("Got an uploaded file: " + item.getFieldName() +
                            ", name = " + item.getName());

                    sfieldname = item.getFieldName();
                    sname = item.getName();

                    sctype = item.getContentType();

                    GcsFilename gcsfileName = new GcsFilename(bucketName, sname);

                    GcsFileOptions options = new GcsFileOptions.Builder()
                    .acl("public-read").mimeType(sctype).build();

                    GcsOutputChannel outputChannel =
                            gcsService.createOrReplace(gcsfileName, options);

                    copy(stream, Channels.newOutputStream(outputChannel));

                    res.sendRedirect("/");
                }
            }
        } catch (Exception ex) {
            throw new ServletException(ex);
        }
    }

    private void copy(InputStream input, OutputStream output) throws IOException {
        try {
          byte[] buffer = new byte[BUFFER_SIZE];
          int bytesRead = input.read(buffer);
          while (bytesRead != -1) {
            output.write(buffer, 0, bytesRead);
            bytesRead = input.read(buffer);
          }
        } finally {
          input.close();
          output.close();
        }
      }

}

I tried also to set the maximumSize of the Upload using upload.setMaxSize(-1); or changing the BUFFER_SIZE from 2*1024*1024 into 200*1024*1024, but the issue stil occur. To be more specific, when the uploading reach the 100% I receive this message on the webpage:

Error: Request Entity Too Large Your client issued a request that was too large.

How can i fix that using JAVA and Google Cloud Storage Client Library for Java? (I'm not going to change drastically the Project with other Programming Languages)

Could you please help me to find a solution? Thank you so much!

Aerox
  • 669
  • 1
  • 13
  • 28

2 Answers2

6

App Engine request limit is 32Mb. That's why your uploads are failing when you send a file > 32Mb. Checkout Quotas and Limits section.

You have two options for uploading files > 32Mb:

Or you could just use Google Drive and store only doc IDs in the datastore :)

alex
  • 2,450
  • 16
  • 22
  • https://developers.google.com/storage/ This link and others don't talk about request limit for Google Cloud Storage, otherwise there would not be any solution to do an Upload. Am i right? – Aerox Apr 18 '14 at 09:13
  • This is not a limit of Cloud Storage but the limit of a *request to App Engine*. See this for more details on limit that you're hitting: https://developers.google.com/appengine/docs/java/#Java_Quotas_and_limits – alex Apr 18 '14 at 09:15
  • If i'm going to use Blobstore there would be problems of duplicate object (in Blobstore and Google Cloud Storage inside buckets). As the "documentation" says, You could use BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService(); BlobKey blobKey = blobstoreService.createGsBlobKey( "/gs/" + fileName.getBucketName() + "/" + fileName.getObjectName()); blobstoreService.serve(blobKey, resp); That is not sufficient to let it work. They miss all the code in the documentation example. Could you help me please? – Aerox Apr 18 '14 at 09:17
  • Why would there be a duplicate? There won't be any duplicates. You either use a Cloud Storage bucket or Blobstore storage. No duplicates. – alex Apr 18 '14 at 09:19
  • @Aerox it's the same code. You just use createUploadUrl(java.lang.String successPath, UploadOptions uploadOptions) method of the BlobstoreService. The rest of the sample code is exactly the same. – alex Apr 18 '14 at 09:33
  • Do i need to use another Serlvet to serve the SuccessPath written in creatUploadUrl method? – Aerox Apr 18 '14 at 09:38
  • Check this out: https://developers.google.com/appengine/docs/java/blobstore/ There's a complete sample. Just use the other `createUploadUrl(...)` method with options. That's the only difference. – alex Apr 18 '14 at 09:43
  • I've already tried it the last week and it works, but i cannot integrate it with my code. The steps to upload a file on Google cloud Storage are: 1)obtain a filename and a bucket 2)Mandatory: create GCSFileOptions because i'm receiving something from a Form in html and i need to know what kind of file it is (application/zip, application/pdf, ..... and so on) get the content and so on. If i'm going to use createUploadUrl method i'will lose every info of the file and what is very important, the content itself. – Aerox Apr 18 '14 at 10:03
  • Moreover, there is the blobstoreService.serve() method that upload the file on GAE (easy to see in Blob View on Application Dashboard), and the writing method (over a Channel) given an InputStream, to write the file on Google Cloud Storage. So, if i'm going to use both, i will upload a duplicate in different location. – Aerox Apr 18 '14 at 10:06
  • Everyone talk about 32MB limit per Api Call ( http://stackoverflow.com/questions/6898736/upload-file-bigger-than-40mb-to-google-app-engine ), but nobody explain how to do it with many calls. I can't find an example (a simple one) to apply what is written in many documentation concerning "Resumable Uploads". I can find just lines of lines of theory but nothing about practical use. Here another link ( http://cloud.dzone.com/articles/comparing-windows-azure-blob ) talking about theory with no linked examples. I know that it's up to us writing code, but sometimes a little example is appreciated – Aerox Apr 18 '14 at 10:28
  • 1
    @Aerox, I'll see if I can find some examples for you later today. I'll let you know. – alex Apr 18 '14 at 10:30
  • Thank you Alex :-) you're very kind – Aerox Apr 18 '14 at 10:56
  • 1
    @Aerox couldn't find a recent sample so I'm building one for you. Should finish it really soon. – alex Apr 19 '14 at 12:54
  • :-) Thank you very much! I hope your code will work for files greater than 32MB that is what i really need :-( Anyway You're the best ever! – Aerox Apr 19 '14 at 13:38
  • 1
    @Aerox here's the sample app: https://github.com/crhym3/java-blobstore-gcs-sample – alex Apr 19 '14 at 20:55
  • I don't know how to thank you! You're great! I'm a beginner on that, so It's a little bit a challenge for me. Thank you so much! Your sample is very useful and clear. Just a question on the sample: In the way it is written, the object uploaded onto GCS has the ObjectName and not the real fileName. How can i change the code in order to fix that? – Aerox Apr 21 '14 at 19:26
  • I'm not able to fix that problem. If i go on my Google Cloud Storage from the Google Console, it shows me the filname as GcsObjectName something like: L2FwcGhvc3RpbmdfcHfa3SC9ibG9icy9BRW5CMlVvTjdtb200SnV4QU9hSWNCLVB3d0wtazlnSEtDQWlyY0xqR09jZTNGU05rbXV10dJskKSWUTZLbVhwd3FPVjRYeUMzRG5PNGlXRTdBOHZGS1RjS1dfdnNDVzdXdy5RRXQ4czcydWw3ejF0NUZW. I don't know why this is associated as fileName instead of simply "sample.txt". – Aerox May 01 '14 at 16:11
  • Does anyone found a solution to this problem ? – BipinSasi Apr 12 '17 at 12:03
0

I will suggest you to take a look at this great and sample example: http://docs.oracle.com/javaee/6/tutorial/doc/glraq.html A good idea will be to monitor the data stream to the server.

Hope it helps

  • 1
    If you don't know what you are writing, I suggest taking a look at http://stackoverflow.com/questions/21406878/uploading-files-with-primefaces-4-0-and-jsf-2-2-on-google-app-engine because Google App Engine doesn't support Servlet 3.0. So I really appreciate if you will answer with a more specific way, because I've been testing this code for 3 weeks and I'm fairly documented. Since the code you've posted (which i've already seen many days ago) is useless in my case. Could you post a working code for Google App Engine please? Thank you so much man. – Aerox Apr 18 '14 at 09:07