Python webscraping: HTML instead of Javascript function

Question

i am trying to scrape the 4 strings within the red box with python from this URL: https://crypto.post.at/CS2/6XFSGJ

When inspecting the page i can find the values in the html body. When i try to get the html with Beautiful Soup or urllib.request i get the following

<!doctype html>
<html lang="en">

<head>
    <meta charset="utf-8" />
    <link rel="icon" href="/favicon.ico" />
    <meta name="viewport" content="width=device-width,initial-scale=1" />
    <meta name="theme-color" content="#000000" />
    <meta name="description" content="A physcial-digital stamp collectible for post.at - by capacity.at" />
    <link rel="apple-touch-icon" href="/logo192.png" />
    <link rel="manifest" href="/manifest.json" />
    <title>Crypto stamp 2</title>
    <script type="text/javascript">console.log = () => { }, console.warn = () => { }, console.error = () => { }</script>
    <link href="/static/css/7.b1cfa592.chunk.css" rel="stylesheet">
    <link href="/static/css/main.86d5d71a.chunk.css" rel="stylesheet">
</head>

<body><noscript>You need to enable JavaScript to run this dApp.</noscript>
    <div id="root"></div>
    <script>!function (e) { function t(t) { for (var n, a, f = t[0], u = t[1], i = t[2], d = 0, s = []; d < f.length; d++)a = f[d], Object.prototype.hasOwnProperty.call(o, a) && o[a] && s.push(o[a][0]), o[a] = 0; for (n in u) Object.prototype.hasOwnProperty.call(u, n) && (e[n] = u[n]); for (l && l(t); s.length;)s.shift()(); return c.push.apply(c, i || []), r() } function r() { for (var e, t = 0; t < c.length; t++) { for (var r = c[t], n = !0, a = 1; a < r.length; a++) { var u = r[a]; 0 !== o[u] && (n = !1) } n && (c.splice(t--, 1), e = f(f.s = r[0])) } return e } var n = {}, a = { 6: 0 }, o = { 6: 0 }, c = []; function f(t) { if (n[t]) return n[t].exports; var r = n[t] = { i: t, l: !1, exports: {} }; return e[t].call(r.exports, r, r.exports, f), r.l = !0, r.exports } f.e = function (e) { var t = []; a[e] ? t.push(a[e]) : 0 !== a[e] && { 0: 1, 1: 1, 8: 1, 9: 1, 10: 1, 11: 1, 12: 1, 13: 1, 14: 1, 16: 1, 17: 1, 18: 1, 20: 1, 21: 1, 22: 1, 23: 1 }[e] && t.push(a[e] = new Promise((function (t, r) { for (var n = "static/css/" + ({}[e] || e) + "." + { 0: "7429d2cd", 1: "a61c0bf9", 2: "31d6cfe0", 3: "31d6cfe0", 4: "31d6cfe0", 8: "aeac5f21", 9: "6ad86d01", 10: "3bb5f1a4", 11: "9617c16b", 12: "f996d814", 13: "0033f4ad", 14: "52eb52b6", 15: "31d6cfe0", 16: "7d672536", 17: "f6d29a98", 18: "543b35a5", 19: "31d6cfe0", 20: "9aea3428", 21: "9aea3428", 22: "9b3f5997", 23: "e520eab9", 24: "31d6cfe0", 25: "31d6cfe0", 26: "31d6cfe0", 27: "31d6cfe0", 28: "31d6cfe0" }[e] + ".chunk.css", o = f.p + n, c = document.getElementsByTagName("link"), u = 0; u < c.length; u++) { var i = (l = c[u]).getAttribute("data-href") || l.getAttribute("href"); if ("stylesheet" === l.rel && (i === n || i === o)) return t() } var d = document.getElementsByTagName("style"); for (u = 0; u < d.length; u++) { var l; if ((i = (l = d[u]).getAttribute("data-href")) === n || i === o) return t() } var s = document.createElement("link"); s.rel = "stylesheet", s.type = "text/css", s.onload = t, s.onerror = function (t) { var n = t && t.target && t.target.src || o, c = new Error("Loading CSS chunk " + e + " failed.\\n(" + n + ")"); c.code = "CSS_CHUNK_LOAD_FAILED", c.request = n, delete a[e], s.parentNode.removeChild(s), r(c) }, s.href = o, document.getElementsByTagName("head")[0].appendChild(s) })).then((function () { a[e] = 0 }))); var r = o[e]; if (0 !== r) if (r) t.push(r[2]); else { var n = new Promise((function (t, n) { r = o[e] = [t, n] })); t.push(r[2] = n); var c, u = document.createElement("script"); u.charset = "utf-8", u.timeout = 120, f.nc && u.setAttribute("nonce", f.nc), u.src = function (e) { return f.p + "static/js/" + ({}[e] || e) + "." + { 0: "37cb5cd9", 1: "27668167", 2: "1935e8bb", 3: "18c6f29c", 4: "bb39ce13", 8: "cabd6ab7", 9: "8952c698", 10: "2b03fd70", 11: "2d167b3a", 12: "9e30f254", 13: "46916483", 14: "2e15bf0e", 15: "76d9f1c6", 16: "a529361e", 17: "f84f46d1", 18: "685d52a1", 19: "24283f6b", 20: "d2e3f4f7", 21: "35128c16", 22: "13b7793f", 23: "97055897", 24: "69ccb34b", 25: "690ede0a", 26: "329c9c33", 27: "2a536f40", 28: "65db6d43" }[e] + ".chunk.js" }(e); var i = new Error; c = function (t) { u.onerror = u.onload = null, clearTimeout(d); var r = o[e]; if (0 !== r) { if (r) { var n = t && ("load" === t.type ? "missing" : t.type), a = t && t.target && t.target.src; i.message = "Loading chunk " + e + " failed.\\n(" + n + ": " + a + ")", i.name = "ChunkLoadError", i.type = n, i.request = a, r[1](i) } o[e] = void 0 } }; var d = setTimeout((function () { c({ type: "timeout", target: u }) }), 12e4); u.onerror = u.onload = c, document.head.appendChild(u) } return Promise.all(t) }, f.m = e, f.c = n, f.d = function (e, t, r) { f.o(e, t) || Object.defineProperty(e, t, { enumerable: !0, get: r }) }, f.r = function (e) { "undefined" != typeof Symbol && Symbol.toStringTag && Object.defineProperty(e, Symbol.toStringTag, { value: "Module" }), Object.defineProperty(e, "__esModule", { value: !0 }) }, f.t = function (e, t) { if (1 & t && (e = f(e)), 8 & t) return e; if (4 & t && "object" == typeof e && e && e.__esModule) return e; var r = Object.create(null); if (f.r(r), Object.defineProperty(r, "default", { enumerable: !0, value: e }), 2 & t && "string" != typeof e) for (var n in e) f.d(r, n, function (t) { return e[t] }.bind(null, n)); return r }, f.n = function (e) { var t = e && e.__esModule ? function () { return e.default } : function () { return e }; return f.d(t, "a", t), t }, f.o = function (e, t) { return Object.prototype.hasOwnProperty.call(e, t) }, f.p = "/", f.oe = function (e) { throw console.error(e), e }; var u = this["webpackJsonpcreate-capacity-app"] = this["webpackJsonpcreate-capacity-app"] || [], i = u.push.bind(u); u.push = t, u = u.slice(); for (var d = 0; d < u.length; d++)t(u[d]); var l = i; r() }([])</script>
    <script src="/static/js/7.12875f3a.chunk.js"></script>
    <script src="/static/js/main.8d5a6bd9.chunk.js"></script>
</body>

</html>

Here the values in the HTML body are missing and there is just the JavaScript script.

How can i scrape this values?

Thanks

Does this answer your question? [Web-scraping JavaScript page with Python](https://stackoverflow.com/questions/8049520/web-scraping-javascript-page-with-python) — ggorlen, Jun 26 '20 at 20:34

score 1 · Accepted Answer · edited Jun 27 '20 at 07:58

The data is loaded dynamically via JavaScript. You only have to compute token ID from the Stamp Code:

import json
import requests
from bs4 import BeautifulSoup


alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
base_count = len(alphabet)
def base58_decode(s):
    """ Decodes the base58-encoded string s into an integer """
    decoded = 0
    multi = 1
    s = s[::-1]
    for char in s:
        decoded += multi * alphabet.index(char)
        multi = multi * base_count

    return decoded

def get_token_id(stamp_id):
    n = (base58_decode(stamp_id) & 0xffffff)
    return n-150000 if n >= 150000 else n

url = 'https://crypto.post.at/CS2/6XFSGJ'
api_url = 'https://crypto.post.at/api/asset/CS2/{}'.format(get_token_id(url.rsplit('/', maxsplit=1)[-1]))
data = requests.get(api_url).json()

print(json.dumps(data, indent=4))

print('-' * 80)

print('contract_symbol:', data['contract_symbol'])
print('type:', data['type'])
print('asset_id:', data['asset_id'])
print('color:', data['color'])

Prints:

{
    "asset_id": "6XFSGJ",
    "color": "black",
    "contract_address": "0xa7f87E8D193E29bf1eD050Fdd511B79Fe0264d8B",
    "contract_ensname": "cs2.cryptostamp.eth",
    "contract_name": "Crypto stamp Edition 2",
    "contract_symbol": "CS2",
    "image_path": "/CS2/image/llama-black",
    "minted": false,
    "network_id": "1",
    "network_name": "main",
    "owner": "0x76192e78957BD40C01321b7aF61C30D5f214580b",
    "owner_ens": null,
    "sales_source_name": "Physical Sale",
    "sales_source_ocs": false,
    "sales_source_presale": false,
    "shop_deliverable": false,
    "shop_delivery_status": 0,
    "shop_delivery_status_name": "None",
    "token_id": 66619,
    "total_number_issued": 240000,
    "type": "llama",
    "uri": "https://crypto.post.at/CS2/meta/66619",
    "used_in_upgrade": false
}
--------------------------------------------------------------------------------
contract_symbol: CS2
type: llama
asset_id: 6XFSGJ
color: black

Python webscraping: HTML instead of Javascript function

1 Answers1