Excel VBA - Split to tabs, runs out of memory

Question

I'm trying to split 700,000 rows into about 27 different tabs, based on manager name. This is obviously a large amount of data and excel runs out of memory and only manages to put across about 100 lines into 1 tab

Does anyone have any idea on how to make the code below more efficient or a different way of getting around running out of memory

Maybe sorting the data first and then cutting and pasting into their own tabs? I'm not sure

Current code:

Sub parse_data()
Dim lr As Long
Dim ws As Worksheet
Dim vcol, i As Long
Dim icol As Long
Dim myarr As Variant
Dim title As String
Dim titlerow As Integer
vcol = 19
Set ws = Sheets("FCW")
lr = ws.Cells(ws.Rows.Count, vcol).End(xlUp).Row
title = "A1:T1"
titlerow = ws.Range(title).Cells(1).Row
icol = ws.Columns.Count
ws.Cells(1, icol) = "Unique"
For i = 2 To lr
On Error Resume Next
If ws.Cells(i, vcol) <> "" And Application.WorksheetFunction.Match(ws.Cells(i, vcol), ws.Columns(icol), 0) = 0 Then
ws.Cells(ws.Rows.Count, icol).End(xlUp).Offset(1) = ws.Cells(i, vcol)
End If
Next
myarr = Application.WorksheetFunction.Transpose(ws.Columns(icol).SpecialCells(xlCellTypeConstants))
ws.Columns(icol).Clear
For i = 2 To UBound(myarr)
ws.Range(title).AutoFilter field:=vcol, Criteria1:=myarr(i) & ""
If Not Evaluate("=ISREF('" & myarr(i) & "'!A1)") Then
Sheets.Add(after:=Worksheets(Worksheets.Count)).Name = myarr(i) & ""
Else
Sheets(myarr(i) & "").Move after:=Worksheets(Worksheets.Count)
End If
ws.Range("A" & titlerow & ":A" & lr).EntireRow.Copy Sheets(myarr(i) & "").Range("A1")
Sheets(myarr(i) & "").Columns.AutoFit
Next
ws.AutoFilterMode = False
ws.Activate
End Sub

Answer 1

Wow. Lots and lots of comments here. @OP, did you ever get this working? If you are still looking for a solution, try this.

Sub Copy_To_Worksheets()
'Note: This macro use the function LastRow
    Dim My_Range As Range
    Dim FieldNum As Long
    Dim CalcMode As Long
    Dim ViewMode As Long
    Dim ws2 As Worksheet
    Dim Lrow As Long
    Dim cell As Range
    Dim CCount As Long
    Dim WSNew As Worksheet
    Dim ErrNum As Long

    'Set filter range on ActiveSheet: A1 is the top left cell of your filter range
    'and the header of the first column, D is the last column in the filter range.
    'You can also add the sheet name to the code like this :
    'Worksheets("Sheet1").Range("A1:D" & LastRow(Worksheets("Sheet1")))
    'No need that the sheet is active then when you run the macro when you use this.
    Set My_Range = Range("A1:D" & LastRow(ActiveSheet))
    My_Range.Parent.Select

    If ActiveWorkbook.ProtectStructure = True Or _
       My_Range.Parent.ProtectContents = True Then
        MsgBox "Sorry, not working when the workbook or worksheet is protected", _
               vbOKOnly, "Copy to new worksheet"
        Exit Sub
    End If

    'This example filters on the first column in the range(change the field if needed)
    'In this case the range starts in A so Field:=1 is column A, 2 = column B, ......
    FieldNum = 1

    'Turn off AutoFilter
    My_Range.Parent.AutoFilterMode = False

    'Change ScreenUpdating, Calculation, EnableEvents, ....
    With Application
        CalcMode = .Calculation
        .Calculation = xlCalculationManual
        .ScreenUpdating = False
        .EnableEvents = False
    End With
    ViewMode = ActiveWindow.View
    ActiveWindow.View = xlNormalView
    ActiveSheet.DisplayPageBreaks = False

    'Add a worksheet to copy the a unique list and add the CriteriaRange
    Set ws2 = Worksheets.Add

    With ws2
        'first we copy the Unique data from the filter field to ws2
        My_Range.Columns(FieldNum).AdvancedFilter _
                Action:=xlFilterCopy, _
                CopyToRange:=.Range("A1"), Unique:=True

        'loop through the unique list in ws2 and filter/copy to a new sheet
        Lrow = .Cells(Rows.Count, "A").End(xlUp).Row
        For Each cell In .Range("A1:A" & Lrow)

            'Filter the range
            My_Range.AutoFilter Field:=FieldNum, Criteria1:="=" & _
             Replace(Replace(Replace(cell.Value, "~", "~~"), "*", "~*"), "?", "~?")

            'Check if there are no more then 8192 areas(limit of areas)
            CCount = 0
            On Error Resume Next
            CCount = My_Range.Columns(1).SpecialCells(xlCellTypeVisible) _
                     .Areas(1).Cells.Count
            On Error GoTo 0
            If CCount = 0 Then
                MsgBox "There are more than 8192 areas for the value : " & cell.Value _
                     & vbNewLine & "It is not possible to copy the visible data." _
                     & vbNewLine & "Tip: Sort your data before you use this macro.", _
                       vbOKOnly, "Split in worksheets"
            Else
                'Add a new worksheet
                Set WSNew = Worksheets.Add(After:=Sheets(Sheets.Count))
                On Error Resume Next
                WSNew.Name = cell.Value
                If Err.Number > 0 Then
                    ErrNum = ErrNum + 1
                    WSNew.Name = "Error_" & Format(ErrNum, "0000")
                    Err.Clear
                End If
                On Error GoTo 0

                'Copy the visible data to the new worksheet
                My_Range.SpecialCells(xlCellTypeVisible).Copy
                With WSNew.Range("A1")
                    ' Paste:=8 will copy the columnwidth in Excel 2000 and higher
                    ' Remove this line if you use Excel 97
                    .PasteSpecial Paste:=8
                    .PasteSpecial xlPasteValues
                    .PasteSpecial xlPasteFormats
                    Application.CutCopyMode = False
                    .Select
                End With
            End If

            'Show all data in the range
            My_Range.AutoFilter Field:=FieldNum

        Next cell

        'Delete the ws2 sheet
        On Error Resume Next
        Application.DisplayAlerts = False
        .Delete
        Application.DisplayAlerts = True
        On Error GoTo 0

    End With

    'Turn off AutoFilter
    My_Range.Parent.AutoFilterMode = False

    If ErrNum > 0 Then
        MsgBox "Rename every WorkSheet name that start with ""Error_"" manually" _
             & vbNewLine & "There are characters in the name that are not allowed" _
             & vbNewLine & "in a sheet name or the worksheet already exist."
    End If

    'Restore ScreenUpdating, Calculation, EnableEvents, ....
    My_Range.Parent.Select
    ActiveWindow.View = ViewMode
    With Application
        .ScreenUpdating = True
        .EnableEvents = True
        .Calculation = CalcMode
    End With

End Sub


Function LastRow(sh As Worksheet)
    On Error Resume Next
    LastRow = sh.Cells.Find(What:="*", _
                            After:=sh.Range("A1"), _
                            Lookat:=xlPart, _
                            LookIn:=xlValues, _
                            SearchOrder:=xlByRows, _
                            SearchDirection:=xlPrevious, _
                            MatchCase:=False).Row
    On Error GoTo 0
End Function

I just tested the functionality by putting =randbetween(1,27) from A1:A700000. The script did everything in less than 30 seconds on my very old ThinkPad with 12GB RAM.

Answer 2

Edit2: Added a loop over manager names stored in a string.

i. In general, turning off screen updating in Excel can speed things up.

    On Error Goto skpError
    Application.ScreenUpdating = False    

    ' your code....

skpError:
    Application.ScreenUpdating = True

ii. If you consider a major overhaul, the following could provide a starting point.

I used simplified sample data like this

manager revenue
Henry   500
Henry   500
Willy   500
Willy   500
Billy   500
Billy   500

In short, it does the following:

• it reads your data into a recordset
• it filters the recordset based on the manager-name
• it copies the records from the recordset to the sheet with the manager-name
• since it doens't explicitly loop every row, it should perform considerably faster than what you had so far

Hope that helps!

Sub WorkWithRecordset()

Dim ws As Worksheet
Dim iCols As Integer

' 1. Reading all the data into a recordset
Dim xlXML As Object
Dim rst As Object

Set rst = CreateObject("ADODB.Recordset")
Set xlXML = CreateObject("MSXML2.DOMDocument")

xlXML.LoadXML ThisWorkbook.Sheets("Data").UsedRange.Value(xlRangeValueMSPersistXML)
rst.Open xlXML

' 2. manager names -  we could also put those into a recordset (similar to above)
'    for showing reasons i use an array here
'    note: i use 2 Variant variables, so I can loop over the arrays-entries without using LBOUND() to UBOUND()

 Dim varManager As Variant
 varManager = Split("Billy;Willy;Henry", ";")


 ' 3. loop over the managers
 Dim manager As Variant
 For Each manager In varManager

    ' set the outputsheet
    Set ws = ThisWorkbook.Sheets(manager)

    ' set the filter on managername
    rst.Filter = "manager = '" & manager & "'"


    With ws

        ' Print the headers
        For iCols = 0 To rst.Fields.Count - 1
            .Cells(1, iCols + 1).Value = rst.Fields(iCols).Name
        Next

        ' Print the data
        .Range("A2").CopyFromRecordset rst

    End With

    ' delete the filter
    rst.Filter = ""

Next manager
' end of manager-loop


Debug.Print "Done. Time " & Now

End Sub


Function GetRecordset(rng As Range) As Object

'Recordset ohne Connection:
'https://usefulgyaan.wordpress.com/2013/07/11/vba-trick-of-the-week-range-to-recordset-without-making-connection/

Dim xlXML As Object
Dim rst As Object

Set rst = CreateObject("ADODB.Recordset")
Set xlXML = CreateObject("MSXML2.DOMDocument")
xlXML.LoadXML rng.Value(xlRangeValueMSPersistXML)

rst.Open xlXML

Set GetRecordset = rst

End Function

Note:

a) the code assumes that there are existing, empty sheets called "Henry", "Billy", "Willy"

b) with 27 sheets you could create manager-sheets dynamically, if they don't already exist

c) i copied the entire rows. if you only need a selection of fields, you could still loop the filtered recordset and access single fields with something like rst!manager

Answer 3

My test stub has 700,000 Rows and 20 Columns of data, 100MB on disk. It takes 6.5 Seconds to parse the data into 27 different worksheets. I'm pretty happy with the results considering it takes 26 Seconds to save the file.

Class Module: ManagerClass

Option Explicit
'Adjust MAXROWS if any Manage will have more than 60000
Private Const MAXROWS As Long = 60000
Private Data
Private m_Manager As String
Private m_ColumnCount As Integer
Private m_Header As Range
Private x As Long
Private y As Integer

Public Sub Init(ColumnCount As Integer, Manager As String, Header As Range)
    m_Manager = Manager
    m_ColumnCount = ColumnCount
    Set m_Header = Header
    ReDim Data(1 To MAXROWS, 1 To ColumnCount)
    x = 1
End Sub

Public Sub Add(Datum As Variant)
    y = y + 1
    If y > m_ColumnCount Then
        y = 1
        x = x + 1
    End If
    Data(x, y) = Datum
End Sub

Private Sub Class_Terminate()
    Dim wsMGR As Worksheet
    If Evaluate("=ISREF('" & m_Manager & "'!A1)") Then
        Set wsMGR = Worksheets(m_Manager)
        wsMGR.Cells.Clear
    Else
        Set wsMGR = Sheets.Add(after:=Worksheets(Worksheets.Count))
        wsMGR.Name = m_Manager
    End If
    wsMGR.Range(m_Header.Address) = m_Header
    wsMGR.Range("A2").Resize(x, m_ColumnCount).Value = Data
End Sub

Standard Module: ParseData

Sub ParseData()
    Application.ScreenUpdating = False
    Application.Calculation = xlCalculationManual

    Const MGRCOLUMN As Integer = 19
    Const HEADERROW As String = "A1:T1"
    Dim Data, MGRData
    Dim key As String
    Dim MGRClass As ManagerClass
    Dim x As Long, y As Long
    Dim dicMGR As Object
    Set dicMGR = CreateObject("Scripting.Dictionary")

    Dim lastRow As Long, z As Long, z2 As Long

    With Sheets("FCW")
        lastRow = .Cells(.Rows.Count, MGRCOLUMN).End(xlUp).Row

        For z = 2 To lastRow Step 10000
            z2 = IIf(z + 10000 > lastRow, lastRow, z + 10000)

            Data = .Range(Cells(z, 1), .Cells(z2, MGRCOLUMN + 1))
            For x = 1 To UBound(Data, 2)
                key = Data(x, MGRCOLUMN)
                If Not dicMGR.Exists(key) Then
                    Set MGRClass = New ManagerClass
                    MGRClass.Init UBound(Data, 2), key, .Range(HEADERROW)
                    dicMGR.Add key, MGRClass
                End If

                For y = 1 To UBound(Data, 2)
                    dicMGR(key).Add Data(x, y)
                Next
            Next
        Next
    End With

    Application.Calculation = xlCalculationAutomatic
    Application.ScreenUpdating = False
End Sub