2

In my Facebook Video Downloader android application i want to show video resolutions like SD, HD with size. Currently i am using InputStreamReader and Pattern.compile method to find SD and HD URL of video. This method rarely gets me HD link of videos and provides only SD URL which can be downloaded.

Below is my code of link parsing

fun linkParsing(url: String, loaded: (item: DownloadItem) -> Unit) {
    val showLogs: Boolean = true
    Log.e("post_url", url)
    return try {
        val getUrl = URL(url)
        val urlConnection =
            getUrl.openConnection() as HttpURLConnection
        var reader: BufferedReader? = null
        urlConnection.setRequestProperty("User-Agent", POST_USER_AGENT)
        urlConnection.setRequestProperty("Accept", "*/*")
        val streamMap = StringBuilder()
        try {
            reader =
                BufferedReader(InputStreamReader(urlConnection.inputStream))
            var line: String?
            while (reader.readLine().also {
                    line = it
                } != null) {
                streamMap.append(line)
            }
        } catch (E: Exception) {
            E.printStackTrace()
            reader?.close()
            urlConnection.disconnect()
        } finally {
            reader?.close()
            urlConnection.disconnect()
        }
        if (streamMap.toString().contains("You must log in to continue.")) {
        } else {
            val metaTAGTitle =
                Pattern.compile("<meta property=\"og:title\"(.+?)\" />")
            val metaTAGTitleMatcher = metaTAGTitle.matcher(streamMap)
            val metaTAGDescription =
                Pattern.compile("<meta property=\"og:description\"(.+?)\" />")
            val metaTAGDescriptionMatcher =
                metaTAGDescription.matcher(streamMap)
            var authorName: String? = ""
            var fileName: String? = ""
            if (metaTAGTitleMatcher.find()) {
                var author =
                    streamMap.substring(metaTAGTitleMatcher.start(), metaTAGTitleMatcher.end())
                Log.e("Extractor", "AUTHOR :: $author")
                author = author.replace("<meta property=\"og:title\" content=\"", "")
                    .replace("\" />", "")
                authorName = author
            } else {
                authorName = "N/A"
            }
            if (metaTAGDescriptionMatcher.find()) {
                var name = streamMap.substring(
                    metaTAGDescriptionMatcher.start(),
                    metaTAGDescriptionMatcher.end()
                )
                Log.e("Extractor", "FILENAME :: $name")
                name = name.replace("<meta property=\"og:description\" content=\"", "")
                    .replace("\" />", "")
                fileName = name
            } else {
                fileName = "N/A"
            }
            val sdVideo =
                Pattern.compile("<meta property=\"og:video\"(.+?)\" />")
            val sdVideoMatcher = sdVideo.matcher(streamMap)
            val imagePattern =
                Pattern.compile("<meta property=\"og:image\"(.+?)\" />")
            val imageMatcher = imagePattern.matcher(streamMap)
            val thumbnailPattern =
                Pattern.compile("<img class=\"_3chq\" src=\"(.+?)\" />")
            val thumbnailMatcher = thumbnailPattern.matcher(streamMap)
            val hdVideo = Pattern.compile("(hd_src):\"(.+?)\"")
            val hdVideoMatcher = hdVideo.matcher(streamMap)
            val facebookFile = DownloadItem()
            facebookFile?.author = authorName
            facebookFile?.filename = fileName
            facebookFile?.postLink = url
            if (sdVideoMatcher.find()) {
                var vUrl = sdVideoMatcher.group()
                vUrl = vUrl.substring(8, vUrl.length - 1) //sd_scr: 8 char
                facebookFile?.sdUrl = vUrl
                facebookFile?.ext = "mp4"
                var imageUrl = streamMap.substring(sdVideoMatcher.start(), sdVideoMatcher.end())
                imageUrl = imageUrl.replace("<meta property=\"og:video\" content=\"", "")
                    .replace("\" />", "").replace("&amp;", "&")
                Log.e("Extractor", "FILENAME :: NULL")
                Log.e("Extractor", "FILENAME :: $imageUrl")
                facebookFile?.sdUrl = URLDecoder.decode(imageUrl, "UTF-8")
                if (showLogs) {
                    Log.e("Extractor", "SD_URL :: Null")
                    Log.e("Extractor", "SD_URL :: $imageUrl")
                }
                if (thumbnailMatcher.find()) {
                    var thumbNailUrl =
                        streamMap.substring(thumbnailMatcher.start(), thumbnailMatcher.end())
                    thumbNailUrl = thumbNailUrl.replace("<img class=\"_3chq\" src=\"", "")
                        .replace("\" />", "").replace("&amp;", "&")
                    Log.e("Extractor", "Thumbnail :: NULL")
                    Log.e("Extractor", "Thumbnail :: $thumbNailUrl")
                    facebookFile?.thumbNailUrl = URLDecoder.decode(thumbNailUrl, "UTF-8")
                }

            }
            if (hdVideoMatcher.find()) {
                var vUrl1 = hdVideoMatcher.group()
                vUrl1 = vUrl1.substring(8, vUrl1.length - 1) //hd_scr: 8 char
                facebookFile?.hdUrl = vUrl1

                if (showLogs) {
                    Log.e("Extractor", "HD_URL :: Null")
                    Log.e("Extractor", "HD_URL :: $vUrl1")
                }

            } else {
                facebookFile?.hdUrl = null
            }
            if (imageMatcher.find()) {
                var imageUrl =
                    streamMap.substring(imageMatcher.start(), imageMatcher.end())
                imageUrl = imageUrl.replace("<meta property=\"og:image\" content=\"", "")
                    .replace("\" />", "").replace("&amp;", "&")
                Log.e("Extractor", "FILENAME :: NULL")
                Log.e("Extractor", "FILENAME :: $imageUrl")
                facebookFile?.imageUrl = URLDecoder.decode(imageUrl, "UTF-8")
            }
            if (facebookFile?.sdUrl == null && facebookFile?.hdUrl == null) {
            }
            loaded(facebookFile!!)
        }
    } catch (e: Exception) {
        e.printStackTrace()
    }
}

I want to implement a feature where i can show different Resolutions with Sizes as shown in this image.

enter image description here

Please note that i have tested my linkParsing method with videos that has HD URL but it gives only SD URL.

This a sample video link: https://fb.watch/aENyxV7gxs/

How this can be done? I am unable to find any proper method or GitHub library for this.

VC.One
  • 14,790
  • 4
  • 25
  • 57
Nayab
  • 112
  • 14

1 Answers1

3

Found a solution for this so posting as answer.

This can be done by extracting Page Source of a webpage and then parsing that XML and fetching list of BASE URLs.

Steps as follow:

1- Load that specific video URL in Webview and get Page Source inside onPageFinished

private fun webViewSetupNotLoggedIn() {
    webView?.settings?.javaScriptEnabled = true
    webView?.settings?.userAgentString = AppConstants.USER_AGENT
    webView?.settings?.useWideViewPort = true
    webView?.settings?.loadWithOverviewMode = true
    webView?.addJavascriptInterface(this, "mJava")
    webView?.post {
        run {
            webView?.loadUrl(“url of your video")
        }
    }
    object : WebViewClient() {
        override fun shouldOverrideUrlLoading(view: WebView, url: String): Boolean {
            if (url == "https://m.facebook.com/login.php" || url.contains("https://m.facebook.com/login.php")
            ) {
                webView?.loadUrl("url of your video")
            }
            return true
        }

        override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) {
            super.onPageStarted(view, url, favicon)
        }
    }
    webView.webChromeClient = object : WebChromeClient() {
        override fun onProgressChanged(view: WebView?, newProgress: Int) {
            super.onProgressChanged(view, newProgress)
            if (progressBarBottomSheet != null) {
                if (newProgress == 100) {
                    progressBarBottomSheet.visibility = View.GONE
                } else {
                    progressBarBottomSheet.visibility = View.VISIBLE
                }
                progressBarBottomSheet.progress = newProgress
            }
        }
    }
    webView?.webViewClient = object : WebViewClient() {
        override fun onPageFinished(view: WebView?, url: String?) {
            try {
                if (webView?.progress == 100) {
                    var original = webView?.originalUrl
                    var post_link = "url of your video"
                    if (original.equals(post_link)) {
                        var listOfResolutions = arrayListOf<ResolutionDetail>()
                        val progressDialog = activity?.getProgressDialog(false)
                        progressDialog?.show()

                        //Fetch resoultions
                        webView.evaluateJavascript(
                            "(function(){return window.document.body.outerHTML})();"
                        ) { value ->
                            val reader = JsonReader(StringReader(value))
                            reader.isLenient = true
                            try {
                                if (reader.peek() == JsonToken.STRING) {
                                    val domStr = reader.nextString()
                                    domStr?.let {
                                        val xmlString = it
                                        CoroutineScope(Dispatchers.Main).launch {
                                            CoroutineScope(Dispatchers.IO).async {
                                                try {
                                                    getVideoResolutionsFromPageSource((xmlString)) {
                                                        listOfResolutions = it
                                                    }
                                                } catch (e: java.lang.Exception) {
                                                    e.printStackTrace()
                                                    Log.e("Exception", e.message!!)
                                                }
                                            }.await()
                                            progressDialog?.hide()
                                            if (listOfResolutions.size > 0) {
                                                setupResolutionsListDialog(listOfResolutions)
                                            } else {
                                                Toast.makeText(
                                                    context,
                                                    "No Resolutions Found",
                                                    Toast.LENGTH_SHORT
                                                ).show()
                                            }
                                        }
                                    }
                                }
                            } catch (e: IOException) {
                                e.printStackTrace()
                            } finally {
                                reader.close()
                            }
                        }
                    }
                }
            } catch (ex: Exception) {
                ex.printStackTrace()
            }
            super.onPageFinished(view, url)
        }

        @TargetApi(android.os.Build.VERSION_CODES.M)
        override fun onReceivedError(
            view: WebView?,
            request: WebResourceRequest?,
            error: WebResourceError
        ) {

        }

        @SuppressWarnings("deprecation")
        override fun onReceivedError(
            view: WebView?,
            errorCode: Int,
            description: String?,
            failingUrl: String?
        ) {
            super.onReceivedError(view, errorCode, description, failingUrl)
        }

        override fun onLoadResource(view: WebView?, url: String?) {
            Log.e("getData", "onLoadResource")

            super.onLoadResource(view, url)
        }
    }
}

2- When Page source is fetched parse to get video Resolution URLs

fun getVideoResolutionsFromPageSource(
    pageSourceXmlString: String?,
    finished: (listOfRes: ArrayList<ResolutionDetail>) -> Unit
) {
    //pageSourceXmlString is the Page Source of WebPage of that specific copied video
    //We need to find list of Base URLs from pageSourceXmlString
    //Base URLs are inside an attribute named data-store which is inside a div whose class name starts with  '_53mw;
    //We need to find that div then get data-store which has a JSON as string
    //Parse that JSON and we will get list of adaptationset
    //Each adaptationset has list of representation tags
    // representation is the actual div which contains BASE URLs
    //Note that: BASE URLs have a specific attribute called mimeType
    //mimeType has audio/mp4 and video/mp4 which helps us to figure out whether the url is of an audio or a video
    val listOfResolutions = arrayListOf<ResolutionDetail>()
    if (!pageSourceXmlString?.isEmpty()!!) {
        val document: org.jsoup.nodes.Document = Jsoup.parse(pageSourceXmlString)
        val sampleDiv = document.getElementsByTag("body")
        if (!sampleDiv.isEmpty()) {
            val bodyDocument: org.jsoup.nodes.Document = Jsoup.parse(sampleDiv.html())
            val dataStoreDiv: org.jsoup.nodes.Element? = bodyDocument.select("div._53mw").first()
            val dataStoreAttr = dataStoreDiv?.attr("data-store")
            val jsonObject = JSONObject(dataStoreAttr)
            if (jsonObject.has("dashManifest")) {
                val dashManifestString: String = jsonObject.getString("dashManifest")
                val dashManifestDoc: org.jsoup.nodes.Document = Jsoup.parse(dashManifestString)
                val mdpTagVal = dashManifestDoc.getElementsByTag("MPD")
                val mdpDoc: org.jsoup.nodes.Document = Jsoup.parse(mdpTagVal.html())
                val periodTagVal = mdpDoc.getElementsByTag("Period")
                val periodDocument: org.jsoup.nodes.Document = Jsoup.parse(periodTagVal.html())
                val subBodyDiv: org.jsoup.nodes.Element? = periodDocument.select("body").first()
                subBodyDiv?.children()?.forEach {
                    val adaptionSetDiv: org.jsoup.nodes.Element? =
                        it.select("adaptationset").first()
                    adaptionSetDiv?.children()?.forEach {
                        if (it is org.jsoup.nodes.Element) {
                            val representationDiv: org.jsoup.nodes.Element? =
                                it.select("representation").first()
                            val resolutionDetail = ResolutionDetail()
                            if (representationDiv?.hasAttr("mimetype")!!) {
                                resolutionDetail.mimetype = representationDiv?.attr("mimetype")
                            }
                            if (representationDiv?.hasAttr("width")!!) {
                                resolutionDetail.width =
                                    representationDiv?.attr("width")?.toLong()!!
                            }
                            if (representationDiv?.hasAttr("height")!!) {
                                resolutionDetail.height =
                                    representationDiv.attr("height").toLong()
                            }
                            if (representationDiv?.hasAttr("FBDefaultQuality")!!) {
                                resolutionDetail.FBDefaultQuality =
                                    representationDiv.attr("FBDefaultQuality")
                            }
                            if (representationDiv?.hasAttr("FBQualityClass")!!) {
                                resolutionDetail.FBQualityClass =
                                    representationDiv.attr("FBQualityClass")
                            }
                            if (representationDiv?.hasAttr("FBQualityLabel")!!) {
                                resolutionDetail.FBQualityLabel =
                                    representationDiv.attr("FBQualityLabel")
                            }
                            val representationDoc: org.jsoup.nodes.Document =
                                Jsoup.parse(representationDiv.html())
                            val baseUrlTag = representationDoc.getElementsByTag("BaseURL")
                            if (!baseUrlTag.isEmpty() && !resolutionDetail.FBQualityLabel.equals(
                                    "Source",
                                    ignoreCase = true
                                )
                            ) {
                                resolutionDetail.videoQualityURL = baseUrlTag[0].text()
                                listOfResolutions.add(resolutionDetail)
                            }
                        }
                    }
                }
            }
        }
    }
    finished(listOfResolutions)
}

class ResolutionDetail {
    var width: Long = 0
    var height: Long = 0
    var FBQualityLabel = ""
    var FBDefaultQuality = ""
    var FBQualityClass = ""
    var videoQualityURL = ""
    var mimetype = ""  // [audio/mp4 for audios and video/mp4 for videos]
}

3- Pass videoQualityURL to your video download function and video in that selected resolution will be downloaded.

Nayab
  • 112
  • 14
  • Hello @Nayab. I've done a similar thing to extract available video resolution and I can download them as well, but there is a problem. Each video I download doesn't have sound in it. And I came to know that Facebook provides video and audio separately. So how did you solve this problem, as I can see you have done a similar thing as I'm doing. You must have faced this issue. Kindly provide your solution for this. – Nabeel Ahmed Feb 10 '22 at 13:11
  • 1
    @NabeelAhmed Please refer to this post – Nayab Feb 11 '22 at 12:57
  • https://stackoverflow.com/questions/71080435/mp4parser-audio-video-merged-output-not-playing-in-all-devices – Nayab Feb 11 '22 at 12:57
  • Hi @Nayab, I've seen your post and it seems a lot of work, as you're downloading video and its audio simultaneously and then you merge both video and audio and then you delete those two audio-video files, and only one file remains which is merged one. I was wondering how you are downloading video and audio simultaneously and which downloading library you are using for this purpose. – Nabeel Ahmed Feb 14 '22 at 06:13
  • @Nayab This is fine for the video for which we have a URL. But do you know how to download other videos those are listed as sub-videos on that page. When user select or play another video, we don't know the URL of that video. I have tried other answer using javascript - on click of video it executes and give you video id and SD url of video but that is not working all the time. After few days, have to do clear data and then it works. onLoadResource() method is also calling very first time when we load the video, When we play it again, it is not calling. Any help would be highly appreciated. – Smeet Sep 06 '22 at 05:38
  • @Smeet any luck on this? – Erselan Khan Apr 15 '23 at 01:48
  • @ErselanKhan No dear. – Smeet Apr 17 '23 at 06:07