Compress and send large string as Spring http response

Question

We have a large JSON String sized about 2GB, which is a response to a http request, how do we compress and send this response to http requester using Spring MVC?

Answer 1

Probably, it's not a complete answer, though there are two things that could help in sending a large response.

1) Use a solution based on the streamed response to compress your data with GZIPOutputStream. Check the complete example for this: GZIPResponseWraper.

Note that in such case you won't need to return any value from your controller -- all the data will be written to the HttpServletResponse response.

2) Enable compression on the container level. There is a built-in support for Tomcat (check out this answer: Using GZIP compression with Spring Boot/MVC/JavaConfig with RESTful). For Jetty, there is a special filter that can be enabled: Jetty GzipFilter.

Hope that helps you a bit.

Answer 2

You can also use lz4 compression, it's a open source project you could find here .

here is the maven dependency :

<dependency>
    <groupId>net.jpountz.lz4</groupId>
    <artifactId>lz4</artifactId>
    <version>1.3.0</version>
</dependency>

Here is sample code :

import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
import net.jpountz.lz4.LZ4FastDecompressor;
import net.jpountz.lz4.LZ4SafeDecompressor;

public class Lz4compression {
    public static void main(String[] args) {
        LZ4Factory factory = LZ4Factory.fastestInstance();
        try {
            String response = "{\"reason\":\"blackList Imei Fail\",\"nextmessagecheck\":86400329,\"status\":null,\"count\":null,\"url\":null,\"message\":null,\"pds\":null} <br/> <pre> {\"ApiRequest\":{\"cityName\":null,\"zip\":\"B63\",\"manufacturer\":\"Samsung\",\"longitude\":\"-73.762847\",\"latitude\":\"40.599204\",\"wifi\":0,\"screenSize\":\"1260_1260\",\"deviceUniqueness\":\"serial\",\"networkSubType\":\"\",\"appName\":\"SDK 6.0\",\"email\":\"glm856@gmail.com\",\"dpi\":480,\"sessionId\":\"9c15ce18a2e65326d375409333f73c2f\",\"locale\":\"en_GB\",\"model\":\"message\",\"action\":\"geticon\",\"debug\":1,\"bannerType\":null,\"supports\":null,\"placementType\":null,\"canShowMR\":null,\"cookieEnabled\":null,\"cookieValue\":null,\"doNotTrack\":null,\"uber\":null,\"serverHostName\":\"uttjgvnnncc\",\"geoIpOrgName\":\"lj\",\"geoIpCityName\":\"Nasik\",\"geoPostalCode\":null,\"geoRemoteAddress\":\" bm,,\",\"geoRegionName\":\"ashtra\",\"geoCountryName\":\"India\",\"geoLatitude\":\"\",\"geoLongitude\":\"\",\"metroId\":\"0\",\"lastLocation\":null,\"phoneNumber\":\"\",\"advertisementId\":\"0\",\"idfa\":null,\"udid\":null,\"src\":null,\"mobileWebRequest\":null,\"httpReferrer\":null,\"APIKEY\":\"hhjjha\",\"appId\":5641,\"imei\":\"hkhdkodyhxaxn\",\"imei_sha\":\"adouxnnx\",\"token\":\"hkhk\",\"request_timestamp\":\"214-02-04 10:10:10_USA_GMT\",\"packageName\":\"com.test.sdk\",\"version\":\"1\",\"language\":\"English\",\"country\":\"United Kingdom\",\"statename\":\"Dudley\",\"carriername\":\"T-Mobile\",\"countryname\":\"United Kingdom\",\"carrier\":\"T-Mobile\",\"networkOperator\":\"T-Mobile\",\"phoneModel\":\"Galaxy 5\",\"ipaddress\":\"679797w917w9279\",\"sdkversion\":\"4.02\",\"useragent\":\"Mozilla%2F5.0+%28Linux%3B+U%3B+Android+4.1.2%3B+en-gb%3B+C6602+Build%2F10.1.1.A.1.253%29+AppleWebKit%2F534.30+%28KHTML%2C+like+Gecko%29+Version%2F4.0+Mobile+Safari%2F534.30\",\"android_id\":\"12e88ce60dc4b8a351da547ce1c3e333\",\"android_id_sha\":\"\",\"isTablet\":false,\"SD\":1.0,\"isConnectionFast\":true,\"unknownsource\":0,\"email_md5\":\"d6c7e534c9957785aa4d7d5148de696e\",\"email_sha\":\"1179e12deacc30d6c0a86c4cbc12aed89c304b1f\",\"overwrite_geo_vars\":true,\"isXMLFound\":true,\"qa_test_api\":null,\"campId\":null,\"creatid\":null,\"placement_id\":null},\"actualImei\":\"22d85254bf3996ff9b8aa2fb6b770740\",\"imei\":\"123bbmbmbmbschkcmbcjsb\",\"stateName\":\"Dudley\",\"countryName\":\"United Kingdom\",\"countryId\":251,\"stateId\":100428,\"cityId\":XXXX,\"deviceId\":XXXX,\"deviceType\":\"PHONE\",\"otherDeviceId\":99825,\"otherDeviceType\":\"PHONE\",\"otherDevice\":1,\"carrierId\":5177,\"carrierName\":\"T-Mobile\",\"otherCarrier\":null,\"manufacturerId\":22,\"userAgentDbId\":null,\"ipAddressToLong\":525631588,\"timeStamp\":1435137078,\"postalCode\":\"B63\",\"isFraudIp\":0,\"platformType\":1,\"deviceUniqueness\":\"2\",\"androidId\":\"291f242383dd76d529e59c22c81a7c3ddf220e1f\",\"cityName\":\"Bristol\",\"networkType\":\"UNKNOWN\",\"actualHeight\":1260.00,\"actualWidth\":1260.00,\"orientationMode\":0,\"failedCases\":null,\"imeiLogs\":null,\"packageId\":84527701,\"sdkversion\":\"4.02\",\"osId\":17,\"testAlgorithmId\":null,\"applicationEntity\":{\"whitelisted\":0,\"blacklisted\":1,\"isBrandSafe\":1,\"bannerRefreshInterval\":0,\"msgpushfrequency\":1.0,\"inactiveUsers\":0,\"delaydays\":0,\"adultcontent\":1,\"appUid\":14365,\"appStatus\":\"APP_RUNNING\",\"appUrl\":\"https://play.google.com/store/apps/details?id=com.blue.kick\",\"appName\":\"Free Flick Kick\",\"appCategory\":\"31928928\",\"isValidUrl\":1,\"isProfane\":-1,\"appTypeStatusListInRunningMode\":[3],\"unsupportedTypes\":{},\"creativeTypeBasedClickIntervalMap\":null,\"creativeTypeBasedCloseIntervalMap\":null,\"creativeTypeBasedCloseDisplayDelayMap\":null,\"creativeTypeBasedAdIntervalMap\":null,\"creativeTypeBasedUpdateTimeMap\":null,\"fullscreenFlag\":0,\"installcount\":5,\"appCreatedDate\":1339104499,\"platformType\":1},\"logingData\":null,\"campaignMultiBids\":null,\"campaignBidsMappingData\":null,\"multiBidCampaignHelper\":null,\"sortedCampaignList\":null,\"adTypeList\":[3],\"applicationAdTypeBasedHashmap\":null,\"appTypeStatusListInRunningMode\":null,\"cpiCampaigns\":null,\"cpiCVRCamps\":null,\"cpiEnabled\":false,\"modelCVRData\":null,\"algorithmId\":0,\"randAlgo\":0,\"throttlePatnerWiseFlag\":null,\"partnerId\":null,\"partnerData\":null,\"admobiusData\":null,\"campaignDetails\":null,\"campaignTypeQuery\":null,\"cutOff\":null,\"blockedApps\":null,\"throttleBalance\":null,\"eventId\":16,\"adType\":\"icon\",\"adTypeId\":3,\"campaignType\":\"icon\",\"adOpt\":false,\"smartWall\":null,\"pushGuid\":null,\"timeSlot\":0,\"inneractiveDynamicFloor\":null,\"inneractiveAid\":null,\"inneractiveCampaign\":0,\"inneractiveRequestCounter\":0,\"inneractiveResponseCounter\":0,\"thirdPartyFeedObject\":null,\"longitude\":\"\",\"latitude\":\"\",\"exchangeId\":0,\"sdData\":null,\"osName\":null,\"appName\":null,\"remoteAddress\":\"31.84.128.100\",\"advertisementId\":null,\"browserId\":0,\"siteUrl\":null,\"siteRef\":null,\"nextMsg\":null,\"revenueEvent\":null,\"devPreRevshare\":null,\"nextMessageCheckTime\":86400000,\"supportedPlatformType\":null,\"segments\":null,\"requestCreateTime\":1435137078755,\"iabCategoryId\":null,\"simEnabled\":1,\"errorMessageEnum\":null,\"isOnlyErrorMessageSend\":null,\"requestValidationResponse\":{\"isOrgNotBlocked\":true,\"isNotInFraudIpFcap\":true,\"isNotInFraudImeiFcap\":true,\"isIpNotInBlacklist\":true,\"isValidSdkVerion\":true,\"isValidOsVersion\":true,\"isImeiValid\":true,\"isValidAppId\":true,\"validRequest\":true},\"placementId\":0,\"fallBackPlacementId\":0,\"pdsRandomValue\":0,\"pdsFailed\":false,\"campEmpty\":false,\"globalFcapReached\":false,\"test\":false,\"childAds\":false}";

            byte[] data = response.getBytes("UTF-8");

            // compress data
            LZ4Compressor compressor = factory.highCompressor();
            byte[] compressed = compressor.compress(data);

            // decompress data
            LZ4FastDecompressor decompressor = factory.fastDecompressor();
            LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
            byte[] restored = decompressor2.decompress(compressed,decompressedLength);

          } catch (UnsupportedEncodingException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
}