AIR (As3) - Download large file trough iPad application

Question

First I want to specify that I can't post source code because the project is huge. I'm trying to download a large file (500+ MB) on iPad device. Initially I tried with URLLoader, but than I realized that the iPad devices has a very limited resources regarding memory. Than I thought that the URLStream will download the file in chunks and with FileStream I can save this chunks on the device (like this AS3: URLStream saving files to desktop?), but I was wrong, the device crashes when I try to download a big file because the RAM of the device is not enough (more precisely this becomes too big: System.privateMemory) Does anyone have any idea how to download a file in chunks and is it possible without using "socket connection"?

Thanks in advance.

EDIT: Here is the code that I use (the commented lines are the version in which the FileStream las closed only after the file is downloaded.

    package components.streamDownloader
    {
        import flash.events.Event;
        import flash.events.EventDispatcher;
        import flash.events.IOErrorEvent;
        import flash.events.OutputProgressEvent;
        import flash.events.ProgressEvent;
        import flash.events.SecurityErrorEvent;
        import flash.filesystem.File;
        import flash.filesystem.FileMode;
        import flash.filesystem.FileStream;
        import flash.net.URLRequest;
        import flash.net.URLStream;
        import flash.system.System;
        import flash.utils.ByteArray;


    /**
     * 
     */
    public class StreamDownloader extends EventDispatcher
    {

        [Event(name="DownloadComplete", type="com.tatstyappz.net.DownloadEvent")]

        [Event(name="Error", type="com.tatstyappz.net.DownloadEvent")]


        //--------------------------------------------------------------------------
        //
        //  Constructor
        //
        //--------------------------------------------------------------------------

        public function StreamDownloader()
        {

        }


        //--------------------------------------------------------------------------
        //
        //  Variables
        //
        //--------------------------------------------------------------------------

        private var file:File;

        //private var fileStream:FileStream;

        private var urlRequest:URLRequest;

        private var urlStream:URLStream;

        private var waitingForDataToWrite:Boolean = false;


        //--------------------------------------------------------------------------
        //
        //  API
        //
        //--------------------------------------------------------------------------

        public function download(urlRequest:URLRequest, file:File):void {


            init();

            this.urlRequest = urlRequest;
            this.file = file;
            //fileStream.open(file, FileMode.WRITE);
            urlStream.load(urlRequest);
        }   


        //--------------------------------------------------------------------------
        //
        //  Event handlers
        //
        //--------------------------------------------------------------------------    

        //----------------------------------
        //  urlStream events
        //----------------------------------

        protected function urlStream_openHandler(event:Event):void
        {
            waitingForDataToWrite = false;
            dispatchEvent(event.clone());
        }

        protected function urlStream_progressHandler(event:ProgressEvent):void
        {


            trace("MEMORY:", System.totalMemoryNumber / 1024 / 1024, "MEMORY P:", System.privateMemory / 1024 / 1024, "FREE MEMORY:", System.freeMemory / 1024 / 1024, "PROGRESS:", event.bytesLoaded / event.bytesTotal );




            if(waitingForDataToWrite){
                writeToDisk();
            }       
        }

        protected function urlStream_completeHandler(event:Event):void
        {
            if(urlStream.bytesAvailable > 0)
            {
                writeToDisk();
            }
            //fileStream.close();

            destory();

            dispatchEvent(event.clone());

            // dispatch additional DownloadEvent
            dispatchEvent(new StreamDownloadEvent(StreamDownloadEvent.DOWNLOAD_COMPLETE, urlRequest, file));        
        }

        protected function urlStream_securityErrorHandler(event:SecurityErrorEvent):void
        {
            dispatchEvent(new StreamDownloadEvent(StreamDownloadEvent.ERROR, urlRequest, file, event.errorID.toString()));
            destory();
        }

        protected function urlStream_ioErrorHandler(event:IOErrorEvent):void
        {
            dispatchEvent(new StreamDownloadEvent(StreamDownloadEvent.ERROR, urlRequest, file, event.errorID.toString()));
            destory();
        }   


        //----------------------------------
        //  fileStream events
        //----------------------------------

        protected function fileStream_outputProgressHandler(event:OutputProgressEvent):void
        {
            waitingForDataToWrite = true;
        }   

        protected function fileStream_ioErrorHandler(event:IOErrorEvent):void
        {
            dispatchEvent(new StreamDownloadEvent(StreamDownloadEvent.ERROR, urlRequest, file, event.errorID.toString()));
            destory();
        }   


        //--------------------------------------------------------------------------
        //
        //  Utils
        //
        //--------------------------------------------------------------------------

        private function init():void
        {
            urlStream = new URLStream();
            //fileStream = new FileStream();

            urlStream.addEventListener(Event.OPEN, urlStream_openHandler);
            urlStream.addEventListener(ProgressEvent.PROGRESS, urlStream_progressHandler); 
            urlStream.addEventListener(Event.COMPLETE, urlStream_completeHandler);
            urlStream.addEventListener(IOErrorEvent.IO_ERROR, urlStream_ioErrorHandler);
            urlStream.addEventListener(SecurityErrorEvent.SECURITY_ERROR, urlStream_securityErrorHandler);

            //fileStream.addEventListener(OutputProgressEvent.OUTPUT_PROGRESS, fileStream_outputProgressHandler)
            //fileStream.addEventListener(IOErrorEvent.IO_ERROR, fileStream_ioErrorHandler);        
        }

        private function destory():void
        {
            urlStream.removeEventListener(Event.OPEN, urlStream_openHandler);
            urlStream.removeEventListener(ProgressEvent.PROGRESS, urlStream_progressHandler); 
            urlStream.removeEventListener(Event.COMPLETE, urlStream_completeHandler);
            urlStream.removeEventListener(IOErrorEvent.IO_ERROR, urlStream_ioErrorHandler);
            urlStream.removeEventListener(SecurityErrorEvent.SECURITY_ERROR, urlStream_securityErrorHandler);

            //fileStream.removeEventListener(OutputProgressEvent.OUTPUT_PROGRESS, fileStream_outputProgressHandler)
            //fileStream.removeEventListener(IOErrorEvent.IO_ERROR, fileStream_ioErrorHandler); 

            urlStream = null;
            //fileStream = null;
        }

        private function writeToDisk():void {
            /*var fileData:ByteArray = new ByteArray();
            urlStream.readBytes(fileData, 0, urlStream.bytesAvailable);
            fileStream.writeBytes(fileData,0,fileData.length);
            waitingForDataToWrite = false;*/

            var bytes:ByteArray = new ByteArray();
            urlStream.readBytes( bytes );

            var fs:FileStream = new FileStream();
            fs.open( file, FileMode.APPEND );
            fs.writeBytes( bytes );
            fs.close();
        }




    }
    }

Answer 1

As I said in my comment to csomakk, I have successfully downloaded 300+ MB files through AIR for desktop, iOS, and Android using the URLStream chunking method.

Pseudo code:

var stream:URLStream = new URLStream();
stream.addEventListener( PROGRESS, progressHandler );
stream.addEventListener( COMPLETE, completeHandler );
stream.load( url );

private function progressHandler( e:ProgressEvent ):void {
    this.writeDataToDisk();
}

private function completeHandler( e:Event ):void {
    this.writeDataToDisk();
}

private function writeDataToDisk():void {
    var bytes:ByteArray = new ByteArray();
    this.stream.readBytes( bytes );

    var fs:FileStream = new FileStream();
    fs.open( file, FileMode.APPEND );
    fs.writeBytes( bytes );
    fs.close();
}

That basic logic works and works just fine up to 300MB (and likely further. Though I should have tested that, now that I think about it). That was written fairly quickly so there may be some errors and I definitely shorthanded a few things, but you get the idea.

If this does not work, we need a few things from you:

1. Post any errors
2. trace out file.size / 1024 / 1024 + "MB" after the fs.close() and see how far it gets before crashing
3. trace out System.memory / 1024 / 1024 + "MB" after thefs.close()` so we can monitor the memory usage

For 2 and 3, we should only need the last trace statements before the crash occurs.

Alternatively, you should know that you won't be able to do anything with that 500MB file in the application. Flash simply will not load it because of its size. The only reason I managed to get away with my 300MB video files is that we were streaming them from disk, not storing the entire things into memory.

Answer 2

Since I am not allowed to comment under Josh's answer for some reason, I am adding my version as a separate answer. But it is heavily based on his suggestion. The code is also available with GitHub at: https://github.com/shishenkov/ActionscriptClasses/blob/master/us/flashmx/net/LargeFileLoader.as

The prerequisite - URLStream is an awesome class but it has a glitch in it that causes a memory leak/buildup to prevent large files from loading properly. The class I'm sharing here has been tested and was able to download a sequence of 1.5GB files into an iPad Air 2 (64GB) without an issue. I assume that larger files will be OK too, since it actually overcomes the RAM storage limitation (before the glitch fix it was crashing at around 200MB).

The glitch - the raw data bytearray where you copy over the loaded bytes never gets disposed of by GC (as noted here: http://blogs.splunk.com/2007/11/16/flashas3-urlstream-memory-leak/) so, the workaround is to implement a class that uses the Josh's technique and makes sure that the bytes are disposed off after the write.

... here's the code (NOTE: this is still pre-production):

package us.flashmx.net 
{
    import flash.events.ErrorEvent;
    import flash.events.Event;
    import flash.events.EventDispatcher;
    import flash.events.HTTPStatusEvent;
    import flash.events.IEventDispatcher;
    import flash.events.IOErrorEvent;
    import flash.events.ProgressEvent;
    import flash.events.SecurityErrorEvent;
    import flash.filesystem.File;
    import flash.filesystem.FileMode;
    import flash.filesystem.FileStream;
    import flash.net.URLRequest;
    import flash.net.URLStream;
    import flash.system.System;
    import flash.utils.ByteArray;

    /**
     * ...
     * @author  Nick Shishenkov <n@vc.am>
     */
    public class LargeFileLoader extends EventDispatcher 
    {
        private var _url:String             = "";
        private var _filePath:String        = "";
        private var _fileStream:FileStream  = new FileStream;
        private var _urlStream:URLStream    = new URLStream;
        private var _localFile:File;
        private var _bytesLoaded:Number;

        public function LargeFileLoader() 
        {
            super(null);

            //
            _urlStream.addEventListener(Event.OPEN, _onOpen, false, int.MIN_VALUE, true);
            _urlStream.addEventListener(ProgressEvent.PROGRESS, _onProgress, false, int.MIN_VALUE, true);
            _urlStream.addEventListener(Event.COMPLETE, _onComplete, false, int.MIN_VALUE, true);
            _urlStream.addEventListener(IOErrorEvent.IO_ERROR, _onError, false, int.MIN_VALUE, true);
            _urlStream.addEventListener(SecurityErrorEvent.SECURITY_ERROR, _onSecurityError, false, int.MIN_VALUE, true);
            _urlStream.addEventListener(HTTPStatusEvent.HTTP_RESPONSE_STATUS, _onHTTPStatus, false, int.MIN_VALUE, true);
            _urlStream.addEventListener(HTTPStatusEvent.HTTP_STATUS, _onHTTPStatus, false, int.MIN_VALUE, true);
        }

        private function _onHTTPStatus(e:HTTPStatusEvent):void 
        {
            dispatchEvent(e.clone());
        }

        public function load(remoteURL:String, localPath:String, overwrite:Boolean = true):void
        {
            _url        = remoteURL;
            _filePath   = localPath;
            //
            _localFile      = new File(_filePath);
            _bytesLoaded    = 0;

            //
            if (overwrite && _localFile.exists)
            {
                _localFile.deleteFile();
            }
            //
            _urlStream.load(new URLRequest(url));
            _fileStream.open(_localFile, FileMode.APPEND);
        }

        private function _onOpen(e:Event):void 
        {
            dispatchEvent(e.clone());
        }

        private function _onSecurityError(e:SecurityErrorEvent):void 
        {
            dispatchEvent(e.clone());
        }

        private function _onError(e:IOErrorEvent):void 
        {
            dispatchEvent(new ErrorEvent(ErrorEvent.ERROR, false, false, e.text));
        }

        private function _onProgress(e:ProgressEvent):void 
        {
            //
            trace(" -> _onProgress: " + _urlStream.length + " | " + e.bytesLoaded + " / " + e.bytesTotal);
            //
            _writeStreamBytes();
            //
            dispatchEvent(new ProgressEvent(ProgressEvent.PROGRESS, false, false, e.bytesLoaded, e.bytesTotal));
        }

        private function _onComplete(e:Event):void 
        {
            _writeStreamBytes();
            //
            dispatchEvent(new Event(Event.COMPLETE));
        }

        private function _writeStreamBytes():void
        {
            var bytes:ByteArray = new ByteArray();
            _urlStream.readBytes( bytes );
            _fileStream.writeBytes( bytes );

            //
            _bytesLoaded    += bytes.length;

            //clear buffer (if the array stays non-null it will lead to a memmory leak
            bytes   = null;

        }

        public function get url():String 
        {
            return _url;
        }

        public function get filePath():String 
        {
            return _filePath;
        }

        public function get bytesLoaded():Number 
        {
            //_localFile.size;
            return _bytesLoaded;
        }


        public function dispose():void
        {
            try{ _fileStream.close(); }catch (err:Error){};

            //
            try{ _urlStream.close(); }catch (err:Error){};

            //
            _urlStream.removeEventListener(Event.OPEN, _onOpen);
            _urlStream.removeEventListener(ProgressEvent.PROGRESS, _onProgress);
            _urlStream.removeEventListener(Event.COMPLETE, _onComplete);
            _urlStream.removeEventListener(IOErrorEvent.IO_ERROR, _onError);
            _urlStream.removeEventListener(SecurityErrorEvent.SECURITY_ERROR, _onSecurityError);
            _urlStream.removeEventListener(HTTPStatusEvent.HTTP_RESPONSE_STATUS, _onHTTPStatus);
            _urlStream.removeEventListener(HTTPStatusEvent.HTTP_STATUS, _onHTTPStatus);

            //
            _urlStream  = null;
            _fileStream = null;

            //
            System.gc();
        }
    }

}

I ran several device tests with Scout CC and the memory stays down at all times (no buildups whatsoever). I will be testing some older iOS devices later on this week. For the record: I am using Adobe AIR 24.0.0.180

here's a sample use:

package us.flashmx.net 
{
    import flash.display.DisplayObject;
    import flash.events.Event;
    import flash.events.ProgressEvent;

    /**
     * ...
     * @author ...
     */
    public class LargeFileLoader_DEMO extends DisplayObject 
    {
        private var _largeFilesLoader:LargeFileLoader;

        public function LargeFileLoader_DEMO() 
        {
            super();
            //
            init_largeFilesLoader("http://A.Large.File.URL/", "/The/Absolute/Local/Path");
        }

        public function dispose_largeFilesLoader():void
        {
            //
            if (_largeFilesLoader != null)
            {
                //clear listeners
                _largeFilesLoader.removeEventListener(ProgressEvent.PROGRESS, _onFileLoaderProgress);
                _largeFilesLoader.removeEventListener(Event.COMPLETE, _onFileLoaderComplete);
                //dispose
                _largeFilesLoader.dispose();
                //free mem
                _largeFilesLoader   = null;
            }           
        }

        private function init_largeFilesLoader(fURL:String, fPath:String):void
        {
            //
            _largeFilesLoader   = new LargeFileLoader;

            //
            _largeFilesLoader.addEventListener(ProgressEvent.PROGRESS, _onFileLoaderProgress, false, int.MIN_VALUE, true);
            _largeFilesLoader.addEventListener(Event.COMPLETE, _onFileLoaderComplete, false, int.MIN_VALUE, true);

            //
            _largeFilesLoader.load(fURL, fPath);
        }

        private function _onFileLoaderComplete(e:Event):void 
        {
            trace("All done!");
            dispose_largeFilesLoader();
        }

        private function _onFileLoaderProgress(e:ProgressEvent):void 
        {
            _largeFilesLoader.bytesLoaded;
        }
    }

}

... I hope that helps !

cheers -Nick