C# NullReference yet Visual Studio shows object is not null

Question

When I press the stop button on the form in my example code, the driver.Quit() call throws a NullReference exception, but when I hover over the object in the IDE when it breaks on the exception, the object is not null.

Furthermore, driver.Quit() is called within a null check condition.

public partial class ProxyScraperForm : Form
{

    private BindingList<IProxyScraperSite> sites = new BindingList<IProxyScraperSite>();
    private List<IWebDriver> activeDrivers = new List<IWebDriver>();

    public BindingList<IProxyScraperSite> Sites { get { return this.sites; } }

    public ProxyScraperForm()
    {
        InitializeComponent();

        sites.Add(new ProxyScraperSiteUsProxyOrg());
        sites.Add(new ProxyScraperSiteFreeProxyListNet());
        sites.Add(new ProxyScraperSiteFreeProxyListsNet());
        sites.Add(new ProxyScraperSiteHideMyName());
        sites.Add(new ProxyScraperSiteHidester());
        scraperDataGridView.DataSource = sites;
    }

    private void scrapeButton_Click(object sender, EventArgs e)
    {
        foreach (var site in sites)
        {
            Task.Run(async () =>
            {
                var driver = SeleniumUtility.CreateDefaultFirefoxDriver();
                activeDrivers.Add(driver);
                await site.ScrapeAsync(driver);
                driver.Quit();
                activeDrivers.Remove(driver);
            });
        }
    }

    private void stopButton_Click(object sender, EventArgs e)
    {
        foreach (var driver in activeDrivers)
        {
            if (driver != null)
            {
                driver.Quit();
            }
        }

        activeDrivers.Clear();

        foreach (var site in sites)
        {
            site.Status = "Idle";
        }
    }

    private void proxyScraperForm_FormClosing(object sender, FormClosingEventArgs e)
    {
        stopButton.PerformClick();
        this.DialogResult = DialogResult.Cancel;
    }
}

Inside site.ScrapeAsync

public class ProxyScraperSiteHideMyName : ProxyScraperSiteBase
{
    public ProxyScraperSiteHideMyName() { SiteName = "HideMyName"; }

    public override Task ScrapeAsync(IWebDriver driver, PauseOrCancelToken pct = null)
    {
        return Task.Run(async () =>
        {
            try
            {
                Status = "Starting";
                driver.Navigate().GoToUrl("https://hidemy.name/en/proxy-list/");
                var wait = new WebDriverWait(driver, new TimeSpan(0, 0, 30));
                var lastPageLinkElement = wait.Until(extras.ExpectedConditions.ElementIsVisible((By.XPath("/html/body/div[1]/div/section[1]/div/div[4]/ul/li[10]/a"))));
                var totalPages = Int32.Parse(lastPageLinkElement.Text);
                for (int i = 0; i < totalPages; i++)
                {
                    Status = $"Scraping page {i + 1} of {totalPages}";
                    var mc = Regex.Matches(driver.PageSource, RegexUtility.IPv4AndPortWithSeperatorRegexString);
                    foreach (Match m in mc)
                    {
                        try
                        {
                            var proxy = new Proxy(m.Groups[1].Value + m.Groups[2].Value);
                            OnProxyScraped(proxy);
                        }
                        catch (Exception ex)
                        {
                            System.Diagnostics.Debug.WriteLine(ex); ;
                        }
                    }

                    if (pct != null) { await pct.PauseOrCancelIfRequested(); }

                    if (i < totalPages - 1)
                    {
                        var nextButton = wait.Until(extras.ExpectedConditions.ElementToBeClickable(By.XPath("//li[@class='arrow__right']/a")));
                        nextButton.Click();
                    }
                }
                Status = "Completed!";
            }
            catch (Exception ex)
            {
                System.Diagnostics.Debug.WriteLine(ex);
            }
        });
    }
}

Stack trace.

at OpenQA.Selenium.DriverService.Stop() 
at OpenQA.Selenium.DriverService.Dispose(Boolean disposing) 
at OpenQA.Selenium.Remote.DriverServiceCommandExecutor.Dispose(Boolean disposing) 
at OpenQA.Selenium.Remote.DriverServiceCommandExecutor.Execute(Command commandToExecute) 
at OpenQA.Selenium.Remote.RemoteWebDriver.Execute(String driverCommandToExecute, Dictionary`2 parameters) 
at OpenQA.Selenium.Remote.RemoteWebDriver.Dispose(Boolean disposing)
at OpenQA.Selenium.Remote.RemoteWebDriver.Quit() 
at HelperLib.ProxyLib.Scraping.ProxyScraperForm.stopButton_Click(Object sender, EventArgs e) in C:\Users\david\source\repos\HelperLib\HelperLib\ProxyLib\Scraping\ProxyScraperForm.cs:line 55 
at System.Windows.Forms.Control.OnClick(EventArgs e) 
at System.Windows.Forms.Button.OnClick(EventArgs e) 
at System.Windows.Forms.Button.OnMouseUp(MouseEventArgs mevent) 
at System.Windows.Forms.Control.WmMouseUp(Message& m, MouseButtons button, Int32 clicks) 
at System.Windows.Forms.Control.WndProc(Message& m) 
at System.Windows.Forms.ButtonBase.WndProc(Message& m) 
at System.Windows.Forms.Button.WndProc(Message& m) 
at System.Windows.Forms.Control.ControlNativeWindow.OnMessage(Message& m) 
at System.Windows.Forms.Control.ControlNativeWindow.WndProc(Message& m) 
at System.Windows.Forms.NativeWindow.DebuggableCallback(IntPtr hWnd, Int32 msg, IntPtr wparam, IntPtr lparam) 
at System.Windows.Forms.UnsafeNativeMethods.DispatchMessageW(MSG& msg)
at System.Windows.Forms.Application.ComponentManager.System.Windows.Forms.UnsafeNativeMethods.IMsoComponentManager.FPushMessageLoop(IntPtr dwComponentID, Int32 reason, Int32 pvLoopData) 
at System.Windows.Forms.Application.ThreadContext.RunMessageLoopInner(Int32 reason, ApplicationContext context) 
at System.Windows.Forms.Application.ThreadContext.RunMessageLoop(Int32 reason, ApplicationContext context) 
at System.Windows.Forms.Application.RunDialog(Form form) 
at System.Windows.Forms.Form.ShowDialog(IWin32Window owner) 
at System.Windows.Forms.Form.ShowDialog() 
at HelperLib.ProxyLib.Testing.proxyTesterView.AddFromScraperToolStripMenuItem_Click(Object sender, EventArgs e) in C:\Users\david\source\repos\HelperLib\HelperLib\ProxyLib\Testing\ProxyTesterView.cs:line 127 
at System.Windows.Forms.ToolStripItem.RaiseEvent(Object key, EventArgs e) 
at System.Windows.Forms.ToolStripMenuItem.OnClick(EventArgs e) 
at System.Windows.Forms.ToolStripItem.HandleClick(EventArgs e) 
at System.Windows.Forms.ToolStripItem.HandleMouseUp(MouseEventArgs e)
at System.Windows.Forms.ToolStripItem.FireEventInteractive(EventArgs e, ToolStripItemEventType met) 
at System.Windows.Forms.ToolStripItem.FireEvent(EventArgs e, ToolStripItemEventType met) 
at System.Windows.Forms.ToolStrip.OnMouseUp(MouseEventArgs mea) 
at System.Windows.Forms.ToolStripDropDown.OnMouseUp(MouseEventArgs mea)
at System.Windows.Forms.Control.WmMouseUp(Message& m, MouseButtons button, Int32 clicks) 
at System.Windows.Forms.Control.WndProc(Message& m) 
at System.Windows.Forms.ScrollableControl.WndProc(Message& m) 
at System.Windows.Forms.ToolStrip.WndProc(Message& m) 
at System.Windows.Forms.ToolStripDropDown.WndProc(Message& m) 
at System.Windows.Forms.Control.ControlNativeWindow.OnMessage(Message& m) 
at System.Windows.Forms.Control.ControlNativeWindow.WndProc(Message& m) 
at System.Windows.Forms.NativeWindow.DebuggableCallback(IntPtr hWnd, Int32 msg, IntPtr wparam, IntPtr lparam) 
at System.Windows.Forms.UnsafeNativeMethods.DispatchMessageW(MSG& msg)
at System.Windows.Forms.Application.ComponentManager.System.Windows.Forms.UnsafeNativeMethods.IMsoComponentManager.FPushMessageLoop(IntPtr dwComponentID, Int32 reason, Int32 pvLoopData) 
at System.Windows.Forms.Application.ThreadContext.RunMessageLoopInner(Int32 reason, ApplicationContext context) 
at System.Windows.Forms.Application.ThreadContext.RunMessageLoop(Int32 reason, ApplicationContext context) 
at System.Windows.Forms.Application.Run(Form mainForm) 
at TestingApp.Program.Main() in C:\Users\david\source\repos\TestingApp\TestingApp\Program.cs:line 19

If you look at the stacktrace for the Exception, is it occurring at the call to `driver.Quit()` or is it occurring inside that call? Is it possible that the exception is coming from the code you're calling into? — scwagner, Apr 26 '18 at 21:16
@xxbbcc The driver.Quit() call in the stopButton handler, not after the await. — JohnWick, Apr 26 '18 at 21:17
Maybe `driver` is non-null but something inside that method is null? — Dan Wilson, Apr 26 '18 at 21:18
@DanWilson Perhaps, but the code in site.ScrapeAsync is inside of a try catch block which simply prints to Debug.WriteLine on exceptions such as this. — JohnWick, Apr 26 '18 at 21:20
A difference between the source code you use for debugging and the binary could cause this behaviour. Are you sure two are the same? — ZiggZagg, Apr 26 '18 at 21:20
This looks like unsafe concurrency. You're adding and removing from activeDrivers on potentially multiple threads. This could be related to your bug, but it certainly a flaw in the code. — Yuli Bonner, Apr 26 '18 at 21:23
@DavidStampher The problem seems to be inside your `driver.Quit();` call — Joe Phillips, Apr 26 '18 at 21:32
@JoePhillips Yeah, the confusing part for me is the IDE shows the object is not null when I hover over it when it breaks as the exception is thrown, yet paradoxically says it is null. Haven't had that happen with a NullReference exception before. — JohnWick, Apr 26 '18 at 21:33
Nothing in that stack trace suggests "driver" is null. It's INSIDE that call and class where something is going wrong There are still 6 calls made after it calls Quit() — Joe Phillips, Apr 26 '18 at 21:36
Seems to me like this is a selenium question. Perhaps you're not disposing something properly? — Joe Phillips, Apr 26 '18 at 21:37
@JoePhillips It's entirely possible, but I'm having a hard time figuring out what is the root of the problem. I suppose that's why I asked the question. I've posted all relevant code, so hopefully someone here can point me in the right direction, or I wouldn't have had to ask in the first place. — JohnWick, Apr 26 '18 at 21:39
Try and make your code run without `Task`s just as a test to rule out concurrency issues. — JohanP, Apr 26 '18 at 21:43
Could you add sleep 2 seconds before call Quit() and see what is happened? — Tony Dong, Apr 26 '18 at 21:43
@TonyDong The exception changes to System.InvalidOperationException: 'Collection was modified; enumeration operation may not execute.' When I add a 5 second sleep before driver.Quit() in my stopButton handler. — JohnWick, Apr 26 '18 at 21:48
@JohanP I can try that, and if it is a concurrency issue (I also feel this is the problem), how can I fix it? — JohnWick, Apr 26 '18 at 21:49
@DavidStampher I would start with `lock`ing on your `activeDrivers` `List` every time you add and remove from it. Or as @Yuli Bonner suggested. — JohanP, Apr 26 '18 at 21:57
If that doesn't work, I would try wrapping driver.Quit(); in a lock. — Yuli Bonner, Apr 26 '18 at 22:01

score 0 · Answer 1 · answered Apr 26 '18 at 22:04

Here is my selenium testing code, and hope it can help you

var tasks = new List<Task>
        {
            new Task(PickUserFirefox)                
        };

        tasks.ForEach(
            task => task.Start()                
        );
        Task.WaitAll(tasks.ToArray());
    }

    private static void PickUserFirefox()
    {
        FirefoxTesting(StaticRandom.Instance.Next(1000,9999), "abcd");
    }

  private static void FirefoxTesting(int id, string fileIds)
    {
        System.Environment.SetEnvironmentVariable("webdriver.gecko.driver", @"D:\Work\Testing\SelenimDriver\geckodriver.exe");
        IWebDriver driver = new FirefoxDriver();
        driver.Navigate().GoToUrl(URL + id + "/" + fileIds);
        Thread.Sleep(1000);
         try
        {
            //Do something here
        }
        catch (Exception e)
        {
            //Error handle here
        }
        Thread.Sleep(1000);
        IWebElement btn = driver.FindElement(By.Id("btnSubmit"));
        btn.Click();
        Thread.Sleep(1000);
        //driver.Close();         
        driver.Quit();
    }

score 0 · Answer 2 · answered Apr 26 '18 at 23:02

Solved by using a ConcurrentDictionary and wrapping driver.Quit() calls in lock statements.

public partial class ProxyScraperForm : Form
{

    private BindingList<IProxyScraperSite> sites = new BindingList<IProxyScraperSite>();
    private ConcurrentDictionary<string, FirefoxDriver> activeDrivers = new ConcurrentDictionary<string, FirefoxDriver>();
    private object lockObj = new object();

    public BindingList<IProxyScraperSite> Sites { get { return this.sites; } }

    public ProxyScraperForm()
    {
        InitializeComponent();

        sites.Add(new ProxyScraperSiteUsProxyOrg());
        sites.Add(new ProxyScraperSiteFreeProxyListNet());
        sites.Add(new ProxyScraperSiteFreeProxyListsNet());
        sites.Add(new ProxyScraperSiteHideMyName());
        sites.Add(new ProxyScraperSiteHidester());
        scraperDataGridView.DataSource = sites;
    }

    private void scrapeButton_Click(object sender, EventArgs e)
    {
        foreach (var site in sites)
        {
            Task.Run(async () =>
            {
                var driver = SeleniumUtility.CreateDefaultFirefoxDriver();
                activeDrivers.TryAdd(site.SiteName, driver);
                await site.ScrapeAsync(driver);
                lock (lockObj)
                {
                    driver.Quit();
                }
                activeDrivers.TryRemove(site.SiteName, out driver);
            });
        }
    }

    private void stopButton_Click(object sender, EventArgs e)
    {
        foreach (var driver in activeDrivers)
        {
            lock (lockObj)
            {
                driver.Value.Quit();
            }
        }

        activeDrivers.Clear();

        foreach (var site in sites)
        {
            site.Status = "Idle";
        }
    }

    private void proxyScraperForm_FormClosing(object sender, FormClosingEventArgs e)
    {
        stopButton.PerformClick();
        this.DialogResult = DialogResult.Cancel;
    }
}

C# NullReference yet Visual Studio shows object is not null

2 Answers2