Rake Task to parse JSON, some objects might be missing

Question

I made a simple Rake Task to parse some JSON and I'm using it to apply some attributes to a game collection.

To avoid a massive list of JSON in this block the JSON can be previewed here and here - These are from the Steam Store's API returning game details. My rake worked when I was only pulling two pieces of information present on every single record (Assuming it was a success)

desc 'update steam game info'

namespace :update_steam_games do

    task scrape_info: :environment do
     require 'open-uri'
    #require 'nokogiri'
    require 'json'

    games = SteamGame.first(10)
    games.each do |game|
        base_url = "http://store.steampowered.com/api/appdetails?appids=#{game['appid']}"
        puts "base_url: #{base_url}"
        results = open(base_url) { |f|
            result = JSON.parse(f.read)
            puts result.inspect
            next if result[game['appid'].to_s]['success'] == false
            type = result[game['appid'].to_s]['data']['type']
            detailed_description = result[game['appid'].to_s]['data']['detailed_description']
            about_the_game = result[game['appid'].to_s]['data']['about_the_game']
            short_description = result[game['appid'].to_s]['data']['short_description']
            supported_languages = result[game['appid'].to_s]['data']['supported_languages']
            if !result[game['appid'].to_s]['data']['dlc'].nil?
                dlc = result[game['appid'].to_s]['data']['dlc'].flatten
            end
            is_free = result[game['appid'].to_s]['data']['is_free']
            required_age = result[game['appid'].to_s]['data']['required_age']
            genres = result[game['appid'].to_s]['data']['genres'].map { |g| g['description'] }
            developers = result[game['appid'].to_s]['data']['developers']
            publishers = result[game['appid'].to_s]['data']['publishers']
            windows = result[game['appid'].to_s]['data']['platforms']['windows']
            mac = result[game['appid'].to_s]['data']['platforms']['mac']
            metacritic_score = result[game['appid'].to_s]['data']['metacritic']['score']
            metacritic_link = result[game['appid'].to_s]['data']['metacritic']['url']
            #game.update_attributes(genres: genres, detailed_description: detailed_description)

            puts game.inspect
        }
        end
    end
end

The current error I'm getting is NoMethodError: undefined method[]' for nil:NilClass` on line 34 - Which is also the end of the attribute assignment, so I am not sure it's directly related to that line.

I did realize I was trying to store an boolean as an integer and have since fixed it, but am now getting this error.

I've tried wrapping everything into a if XXX .present? wrapper, but then I get the as below

desc 'update steam game info'

namespace :update_steam_games do

    task scrape_info: :environment do
     require 'open-uri'
    #require 'nokogiri'
    require 'json'

    games = SteamGame.first(10)
    games.each do |game|
        base_url = "http://store.steampowered.com/api/appdetails?appids=#{game['appid']}"
        puts "base_url: #{base_url}"
        results = open(base_url) { |f|
            result = JSON.parse(f.read)
            puts result.inspect
            next if result[game['appid'].to_s]['success'] == false
            if type = result[game['appid'].to_s]['data']['type'].present?
                type = result[game['appid'].to_s]['data']['type']
            end
            if detailed_description = result[game['appid'].to_s]['data']['detailed_description'].present?
                detailed_description = result[game['appid'].to_s]['data']['detailed_description']
            end
            if about_the_game = result[game['appid'].to_s]['data']['about_the_game'].present?
                about_the_game = result[game['appid'].to_s]['data']['about_the_game']
            end
            if short_description = result[game['appid'].to_s]['data']['short_description'].present?
                short_description = result[game['appid'].to_s]['data']['short_description']
            end
            if supported_languages = result[game['appid'].to_s]['data']['supported_languages'].present?
                supported_languages = result[game['appid'].to_s]['data']['supported_languages']
            end
            if result[game['appid'].to_s]['data']['dlc'].present?.present?
                    dlc = result[game['appid'].to_s]['data']['dlc'].flatten
            end
            if is_free = result[game['appid'].to_s]['data']['is_free'].present?
                is_free = result[game['appid'].to_s]['data']['is_free']
            end
            if required_age = result[game['appid'].to_s]['data']['required_age'].present?
                required_age = result[game['appid'].to_s]['data']['required_age']
            end
            if genres = result[game['appid'].to_s]['data']['genres'].map { |g| g['description'] }.present?
                genres = result[game['appid'].to_s]['data']['genres'].map { |g| g['description'] }
            end
            if developers = result[game['appid'].to_s]['data']['developers'].present?
                developers = result[game['appid'].to_s]['data']['developers']
            end
            if publishers = result[game['appid'].to_s]['data']['publishers'].present?
                publishers = result[game['appid'].to_s]['data']['publishers']
            end
            if windows = result[game['appid'].to_s]['data']['platforms']['windows'].present?
                windows = result[game['appid'].to_s]['data']['platforms']['windows']
            end
            if mac = result[game['appid'].to_s]['data']['platforms']['mac'].present?
                mac = result[game['appid'].to_s]['data']['platforms']['mac']
            end
            if metacritic_score = result[game['appid'].to_s]['data']['metacritic']['score'].present?
                metacritic_score = result[game['appid'].to_s]['data']['metacritic']['score']
            end
            if metacritic_link = result[game['appid'].to_s]['data']['metacritic']['url'].present?
                metacritic_link = result[game['appid'].to_s]['data']['metacritic']['url']
            end
            #game.update_attributes(genres: genres, detailed_description: detailed_description)

            puts game.inspect
        }
        end
    end
end

But I actually still end up with the same issue, in addition to the fact that I'm unsure how I'm going to even pass it to game.update_attributes without knowing if it even exists.

How do I inspect what is actually the problem, and make this less redundant?

The JSON

In case you don't want to click the link I have provided a sample of the JSON here. I have removed some of it for bloat. The 10 that it starts with is the dynamic ID.

  "10": {
    "success": true,
    "data": {
      "type": "game",
      "name": "Counter-Strike",
      "steam_appid": 10,
      "required_age": 0,
      "is_free": false,
      "detailed_description": "Play the world's number 1 online action game. Engage in an incredibly realistic brand of terrorist warfare in this wildly popular team-based game. Ally with teammates to complete strategic missions. Take out enemy sites. Rescue hostages. Your role affects your team's success. Your team's success affects your role.",
      "about_the_game": "Play the world's number 1 online action game. Engage in an incredibly realistic brand of terrorist warfare in this wildly popular team-based game. Ally with teammates to complete strategic missions. Take out enemy sites. Rescue hostages. Your role affects your team's success. Your team's success affects your role.",
      "short_description": "",
      "supported_languages": "English, French, German, Italian, Spanish, Simplified Chinese, Traditional Chinese, Korean",
      "header_image": "http://cdn.akamai.steamstatic.com/steam/apps/10/header.jpg?t=1447887426",
      "website": null,
      "pc_requirements": {
        "minimum": "\n\t\t\t<p><strong>Minimum:</strong> 500 mhz processor, 96mb ram, 16mb video card, Windows XP, Mouse, Keyboard, Internet Connection<br /></p>\n\t\t\t<p><strong>Recommended:</strong> 800 mhz processor, 128mb ram, 32mb+ video card, Windows XP, Mouse, Keyboard, Internet Connection<br /></p>\n\t\t\t"
      },
      "mac_requirements": {
        "minimum": "Minimum: OS X  Snow Leopard 10.6.3, 1GB RAM, 4GB Hard Drive Space,NVIDIA GeForce 8 or higher, ATI X1600 or higher, or Intel HD 3000 or higher Mouse, Keyboard, Internet Connection"
      },
      "linux_requirements": {
        "minimum": "Minimum: Linux Ubuntu 12.04, Dual-core from Intel or AMD at 2.8 GHz, 1GB Memory, nVidia GeForce 8600/9600GT, ATI/AMD Radeaon HD2600/3600 (Graphic Drivers: nVidia 310, AMD 12.11), OpenGL 2.1, 4GB Hard Drive Space, OpenAL Compatible Sound Card"
      },
      "developers": [
        "Valve"
      ],
      "publishers": [
        "Valve"
      ],
      "price_overview": {
        "currency": "USD",
        "initial": 999,
        "final": 999,
        "discount_percent": 0
      },
      "packages": [
        7
      ],
      "package_groups": [
        {
          "name": "default",
          "title": "Buy Counter-Strike",
          "description": "",
          "selection_text": "Select a purchase option",
          "save_text": "",
          "display_type": 0,
          "is_recurring_subscription": "false",
          "subs": [
            {
              "packageid": 7,
              "percent_savings_text": "",
              "percent_savings": 0,
              "option_text": "Counter-Strike: Condition Zero - $9.99",
              "option_description": "",
              "can_get_free_license": "0",
              "is_free_license": false,
              "price_in_cents_with_discount": 999
            }
          ]
        }
      ],
      "platforms": {
        "windows": true,
        "mac": true,
        "linux": true
      },
      "metacritic": {
        "score": 88,
        "url": "http://www.metacritic.com/game/pc/counter-strike?ftag=MCD-06-10aaa1f"
      }
    }
}

Update

I realized that I can't check on the existance of platform - windows if platform doesn't exist. The code now successfully executes, however, I am having an issue writing since a lot of parameters may or may not be missing on every pass.

What is the proper method to do this instead of the if blocks everywhere?

Update

My code now works after I realize I stupidly put the if block around the variable definement portion.

However, this is still redundant. Is there a proper way to do this, or is this fine?

check this https://stackoverflow.com/a/6225321/5113690 – m3characters Jul 26 '17 at 00:42 — m3characters, Jul 26 '17 at 00:42

score 0 · Answer 1 · answered Jul 26 '17 at 10:55

games = SteamGame.first(10)
games.each do |game|
    base_url = "http://store.steampowered.com/api/appdetails?appids=#{game['appid']}"
    puts "base_url: #{base_url}"
    results = open(base_url) { |f|
        result = JSON.parse(f.read)
        game_appid = game[:appid].to_s
        result_for_appid = result[game_appid]
        result_data = result_for_appid[:data]
        puts result.inspect
        next if result_for_appid[:success] == false
        type = result_data[:type]
        detailed_description = result_data[:detailed_description]
        about_the_game = result_data[:about_the_game]
        short_description = result_data[:short_description]
        supported_languages = result_data[:supported_languages]
        if result_data[:dlc].present?
            dlc = result_data[:dlc].flatten
        end
        is_free = result_data[:is_free]
        required_age = result_data[:required_age]
        # genres = result_data[:genres].map { |g| g[:description] }
        # Your provided JSON didn't contain 'genres' inside 'data'
        developers = result_data[:developers]
        publishers = result_data[:publishers]
        windows = result_data[:platforms][:windows]
        mac = result_data[:platforms][:mac]
        metacritic_score = result_data[:metacritic][:score]
        metacritic_link = result_data[:metacritic][:url]
        # game.update_attributes(genres: genres, detailed_description: detailed_description)
        # Your provided JSON didn't contain 'genres' inside 'data' so no update for this
        game.update_attributes(detailed_description: detailed_description)

        puts game.inspect
    }
    end
end

Few tips for this: DRY - Don't Repeat Yourself You can access json keys as symbols You didn't have "genres' in Your provided JSON You don't have to use IF's - when there's no data You will get: variable = nil. You can later update attributes regarding variables values.

Thanks - Yea the genre things was actually part of my issue on the successful ones. My issue is right now when it gets to any of the columns that are blank, I get a response similar to that of `NoMethodError: undefined method[]' for nil:NilClass`` - I will try with your logic of having data predefined because I hate the if blocks everywhere. — DNorthrup, Jul 26 '17 at 12:01

Rake Task to parse JSON, some objects might be missing

1 Answers1