Convert JSON to CURL cookies then scrape using it

Question

I followed the documentation about cookies but still not working.

In my code below, i have an array and json, so when i tried to convert it to CURL cookies, and placed it in my code, it still not works:

[
{
    "domain": ".bartleby.com",
    "expirationDate": 1670774433.707552,
    "hostOnly": false,
    "httpOnly": false,
    "name": "OptanonConsent",
    "path": "/",
    "sameSite": "lax",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "isGpcEnabled=1&datestamp=Wed+Jun+15+2022+00%3A00%3A33+GMT%2B0800+(China+Standard+Time)&version=6.32.0&isIABGlobal=false&hosts=&consentId=d4c31057-054c-40cc-a24b-95462beba80f&interactionCount=1&landingPath=NotLandingPage&groups=C0001%3A1%2CC0003%3A1%2CBG142%3A0%2CC0002%3A0%2CC0005%3A0%2CC0004%3A0&AwaitingReconsent=false",
    "id": 1
},
{
    "domain": ".www.bartleby.com",
    "expirationDate": 1670774385.651645,
    "hostOnly": false,
    "httpOnly": false,
    "name": "G_ENABLED_IDPS",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "google",
    "id": 2
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1655222715,
    "hostOnly": true,
    "httpOnly": false,
    "name": "accessToken",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "66cbb1942c147c5c0ab49cd02c5fe1191722229e",
    "id": 3
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1657814425,
    "hostOnly": true,
    "httpOnly": false,
    "name": "bartlebyRefreshTokenExpiresAt",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "2022-07-14T16:00:25.189Z",
    "id": 4
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1655308832,
    "hostOnly": true,
    "httpOnly": false,
    "name": "btbHomeDashboardAnimationTriggerDate",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "2022-06-15T16:00:32.720Z",
    "id": 5
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1670774432.724345,
    "hostOnly": true,
    "httpOnly": false,
    "name": "btbHomeDashboardTooltipAnimationCount",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "0",
    "id": 6
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "promotionId",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "",
    "id": 7
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "refreshToken",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "6d4abe8a361878f53a6cfe6a413688168eb8b282",
    "id": 8
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "sku",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "bb999_bookstore",
    "id": 9
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "userId",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "4c28bc2c-1eec-4d2c-b44d-7bfa78216ba3",
    "id": 10
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "userStatus",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "A1",
    "id": 11
}
]

TO:

G_ENABLED_IDPS=google; btbHomeDashboardAnimationTriggerDate=2022-06-15T16:00:32.720Z; btbHomeDashboardTooltipAnimationCount=0; promotionId=; refreshToken=6d4abe8a361878f53a6cfe6a413688168eb8b282; sku=bb999_bookstore; userId=4c28bc2c-1eec-4d2c-b44d-7bfa78216ba3; userStatus=A1; bartlebyRefreshTokenExpiresAt=2022-07-14T17:23:47.098Z; OptanonConsent=isGpcEnabled=1&datestamp=Wed+Jun+15+2022+01%3A24%3A10+GMT%2B0800+(China+Standard+Time)&version=6.32.0&isIABGlobal=false&hosts=&consentId=d4c31057-054c-40cc-a24b-95462beba80f&interactionCount=1&landingPath=NotLandingPage&groups=C0001%3A1%2CC0003%3A1%2CBG142%3A0%2CC0002%3A0%2CC0005%3A0%2CC0004%3A0&AwaitingReconsent=false; endCycleWhenQuestionsRemainingWasClosed=2022-06-19T07:00:00.000Z

The JSON array cookies above really work I applied it on my browser and I successfully logged in, but when I apply it on CURL it does not work.

I am trying to scrape the contents of a certain website, but in order to get the contents, I need to be logged in. So to solve this, I exported my cookies and I will use them on Curl.

Here is my code but did not work because when I run it, it only displays: Not Found This page you were trying to reach at this address doesn't seem to exist, but when you access the $url directly on your browser: https://www.bartleby.com/questions-and-answers/why-is-stack-overflow-so-bad-when-developing-for-the-compiler-we-used-in-class-what-do-modern-system/ff7a730f-e3c5-4baf-9b75-0dbd5f1581fd, it works. Thanks:

<?php

$curl = curl_init();
$url = "https://www.bartleby.com/questions-and-answers/why-is-stack-overflow-so-bad-when-developing-for-the-compiler-we-used-in-class-what-do-modern-system/ff7a730f-e3c5-4baf-9b75-0dbd5f1581fd";
    $cookieFile = '[
{
    "domain": ".bartleby.com",
    "expirationDate": 1670774433.707552,
    "hostOnly": false,
    "httpOnly": false,
    "name": "OptanonConsent",
    "path": "/",
    "sameSite": "lax",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "isGpcEnabled=1&datestamp=Wed+Jun+15+2022+00%3A00%3A33+GMT%2B0800+(China+Standard+Time)&version=6.32.0&isIABGlobal=false&hosts=&consentId=d4c31057-054c-40cc-a24b-95462beba80f&interactionCount=1&landingPath=NotLandingPage&groups=C0001%3A1%2CC0003%3A1%2CBG142%3A0%2CC0002%3A0%2CC0005%3A0%2CC0004%3A0&AwaitingReconsent=false",
    "id": 1
},
{
    "domain": ".www.bartleby.com",
    "expirationDate": 1670774385.651645,
    "hostOnly": false,
    "httpOnly": false,
    "name": "G_ENABLED_IDPS",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "google",
    "id": 2
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1655222715,
    "hostOnly": true,
    "httpOnly": false,
    "name": "accessToken",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "66cbb1942c147c5c0ab49cd02c5fe1191722229e",
    "id": 3
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1657814425,
    "hostOnly": true,
    "httpOnly": false,
    "name": "bartlebyRefreshTokenExpiresAt",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "2022-07-14T16:00:25.189Z",
    "id": 4
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1655308832,
    "hostOnly": true,
    "httpOnly": false,
    "name": "btbHomeDashboardAnimationTriggerDate",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "2022-06-15T16:00:32.720Z",
    "id": 5
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1670774432.724345,
    "hostOnly": true,
    "httpOnly": false,
    "name": "btbHomeDashboardTooltipAnimationCount",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "0",
    "id": 6
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "promotionId",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "",
    "id": 7
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "refreshToken",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "6d4abe8a361878f53a6cfe6a413688168eb8b282",
    "id": 8
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "sku",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "bb999_bookstore",
    "id": 9
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "userId",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "4c28bc2c-1eec-4d2c-b44d-7bfa78216ba3",
    "id": 10
},
{
    "domain": "www.bartleby.com",
    "expirationDate": 1656432015,
    "hostOnly": true,
    "httpOnly": false,
    "name": "userStatus",
    "path": "/",
    "sameSite": "unspecified",
    "secure": false,
    "session": false,
    "storeId": "0",
    "value": "A1",
    "id": 11
}
]';
curl_setopt ($curl, CURLOPT_COOKIEFILE, $cookieFile);
curl_setopt ($curl, CURLOPT_COOKIEJAR, $cookieFile);
curl_setopt($curl, CURLOPT_URL, $url);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);


$output = curl_exec($curl);
$info = curl_getinfo($curl);
$err = curl_error($curl);
$ern = curl_errno($curl);

if ($ern) {
    printf("An error occurred: (%d) %s\n", $ern, $err);
    exit(1);
}
curl_close($curl);

// Debug only.
// var_dump($output);

echo $output;

?>

score -1 · Answer 1 · answered Jun 14 '22 at 19:18

-1

You need to send the cookie as txt file with the proper cookie file format explained here: https://curl.se/docs/http-cookies.html#:~:text=The%20cookie%20file%20format%20is,end%20with%20a%20newline%20character. If you simply send the array it would not work. Thanks

answered Jun 14 '22 at 19:18

Mahedi Hasan

178
1
11

Thanks, But what is the file format of cookies? and the file extension? – ejade Jun 14 '22 at 19:20
The cookie file format is text-based(txt) and stores one cookie per line. Each line that specifies a single cookie consists of seven text fields separated with TAB characters. A valid line must end with a newline character. Fields in the file string example.com - the domain name boolean FALSE - include subdomains string /foobar/ - path boolean TRUE - send/receive over HTTPS only number 1462299217 - expires at - seconds since Jan 1st 1970, or 0 string person - name of the cookie string daniel - value of the cookie – Mahedi Hasan Jun 14 '22 at 19:28

Convert JSON to CURL cookies then scrape using it

1 Answers1