How to improve performance when query many data on ALL contacts from Android ContactsContract.Contacts?

Question

OBJECTIVE

I want to get for each contacts in the user phone the following data

StructuredName.GIVEN_NAME|Phone.NUMBER|Email.DATA|StructuredPostal.CITY

I'm pretty sure I query the ContactsContract.Data table with a pure SQL query but the there is no clear documentation on how to do it. It seems that you can inject SQL in the contentResolver.query but it does not seem to be sustainable.

PROBLEM

My code hereafter works perfectly but is very slow.

Basically,

I get the IDs of ALL contacts from ContactsContract.Contacts and LOOP through it and for each,
SELECT naming data on CommonDataKinds.StructuredName,
SELECT and LOOP phone data on CommonDataKinds.Phone,
SELECT and LOOP email data on CommonDataKinds.Email,
SELECT and LOOP address data on CommonDataKinds.StructuredPostal

However, the many loops are obviously counterproductive in term of performance.

With a 1000 contacts, it makes around 3000 queries.

CODE

// CREATE Content resolver
val resolver: ContentResolver = contentResolver
val cursor = resolver.query(
        ContactsContract.Contacts.CONTENT_URI,
        arrayOf(
                ContactsContract.Contacts._ID
        ),
        null,
        null,
        null
)

if ( cursor != null && cursor.count > 0) {
    // PROGRESSBAR Process
    myProgressBar?.progress = 0
    myProgressBarCircleText?.text = getString(R.string.processing_contacts)
    myProgressBar?.visibility = View.VISIBLE
    myProgressBarCircle?.visibility = View.VISIBLE
    myProgressBarCircleText?.visibility = View.VISIBLE



    // PUT BASIC REQUIRED INFO
    val jsonAllContacts = JSONObject()
    jsonAllContacts.put("source", "2")



    // EXECUTE CODE on another thread to prevent blocking UI
    Thread(Runnable {
        var cursorPosition = 0
        var currentProgress: Int


        Log.e("JSON", "cursor.count: ${cursor.count}")


        // CODE TO EXEC LOOP
        while (cursor.moveToNext()) {

            // Increment cursor for progressBar
            cursorPosition += 1
            currentProgress = ((cursorPosition.toFloat() / cursor.count.toFloat()) * 100).toInt()


            // INIT of jsonObjects
            val jsonEmail = JSONObject()
            val jsonPhone = JSONObject()
            val jsonAddress = JSONObject()
            val jsonCurrentContact = JSONObject()



            /**
             * NAME DETAILS
             */
            val contactID = cursor.getString(cursor.getColumnIndex(ContactsContract.Contacts._ID))



            val nameCur = contentResolver.query(
                    ContactsContract.Data.CONTENT_URI,
                    arrayOf(
                            ContactsContract.CommonDataKinds.StructuredName.GIVEN_NAME,
                            ContactsContract.CommonDataKinds.StructuredName.FAMILY_NAME,
                            ContactsContract.CommonDataKinds.StructuredName.MIDDLE_NAME
                    ),
                    ContactsContract.Data.CONTACT_ID + " = ?" + " AND " + ContactsContract.Data.MIMETYPE + " = ?",
                    arrayOf(
                            contactID,
                            ContactsContract.CommonDataKinds.StructuredName.CONTENT_ITEM_TYPE
                    ),
                    null
            )
            var givenName = ""
            var familyName: String
            var middleName: String
            var fullName = ""

            if ( nameCur != null ) {
                while (nameCur.moveToNext()) {

                    givenName = nameCur.getString(nameCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredName.GIVEN_NAME)) ?: ""
                    middleName = nameCur.getString(nameCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredName.MIDDLE_NAME)) ?: ""
                    familyName = nameCur.getString(nameCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredName.FAMILY_NAME)) ?: ""


                    fullName = if ( middleName != "" && (middleName != familyName) ) {
                        "$middleName $familyName"
                    } else {
                        familyName
                    }
                }

                jsonCurrentContact.put("given", givenName)
                jsonCurrentContact.put("family", fullName)
            }
            nameCur?.close()


            /**
             * PHONE NUMBER
             */
            val phoneCur = contentResolver.query(
                    ContactsContract.CommonDataKinds.Phone.CONTENT_URI,
                    arrayOf(
                            ContactsContract.CommonDataKinds.Phone.TYPE,
                            ContactsContract.CommonDataKinds.Phone.LABEL,
                            ContactsContract.CommonDataKinds.Phone.NUMBER
                    ),
                    ContactsContract.CommonDataKinds.Phone.CONTACT_ID + "=?",
                    arrayOf( contactID ),
                    null
            )

            if ( phoneCur != null && phoneCur.count > 0 ) {
                while (phoneCur.moveToNext()) {
                    val phoneNumType = phoneCur.getString( phoneCur.getColumnIndex(ContactsContract.CommonDataKinds.Phone.TYPE) ) ?: ""
                    val phoneNumLabel = phoneCur.getString( phoneCur.getColumnIndex(ContactsContract.CommonDataKinds.Phone.LABEL) ) ?: ""
                    var label: String
                    val phoneNumber = phoneCur.getString( phoneCur.getColumnIndex(ContactsContract.CommonDataKinds.Phone.NUMBER) ).replace(" ", "") ?: ""

                    //Log.e("JSON", "JSON phoneNum: $phoneNumLabel $phoneNumber")


                    // TRY to get label info
                    label = if ( phoneNumType == "" ) {
                        phoneNumLabel
                    } else {
                        phoneNumType
                    }

                    jsonPhone.put("label", label)
                    jsonPhone.put("number", phoneNumber)
                    jsonCurrentContact.accumulate("phone", jsonPhone)

                }


            }
            phoneCur?.close()


            /**
             * EMAIL
             */
            val emailCur = contentResolver.query(
                    ContactsContract.CommonDataKinds.Email.CONTENT_URI,
                    arrayOf(
                            ContactsContract.CommonDataKinds.Email.LABEL,
                            ContactsContract.CommonDataKinds.Email.DATA
                    ),
                    ContactsContract.CommonDataKinds.Email.CONTACT_ID + "=?",
                    arrayOf(contactID),
                    null
            )
            if ( emailCur != null ) {

                while (emailCur.moveToNext()) {
                    val emailLabel = emailCur.getString(emailCur.getColumnIndex(ContactsContract.CommonDataKinds.Email.LABEL)) ?: ""
                    val email = emailCur.getString(emailCur.getColumnIndex(ContactsContract.CommonDataKinds.Email.DATA)) ?: ""

                    jsonEmail.put("label", emailLabel)
                    jsonEmail.put("email", email)
                    jsonCurrentContact.accumulate("email", jsonEmail)

                }

            }
            emailCur?.close()




            /**
             * ADDRESS
             */
            var street: String
            var city: String
            var postalCode: String
            var state: String
            var country: String
            var label: String
            val addressCur = contentResolver.query(
                    ContactsContract.CommonDataKinds.StructuredPostal.CONTENT_URI,
                    arrayOf(
                            ContactsContract.CommonDataKinds.StructuredPostal.TYPE,
                            ContactsContract.CommonDataKinds.StructuredPostal.STREET,
                            ContactsContract.CommonDataKinds.StructuredPostal.CITY,
                            ContactsContract.CommonDataKinds.StructuredPostal.POSTCODE,
                            ContactsContract.CommonDataKinds.StructuredPostal.REGION,
                            ContactsContract.CommonDataKinds.StructuredPostal.COUNTRY
                    ),
                    ContactsContract.CommonDataKinds.StructuredPostal.CONTACT_ID + "=" + contactID,
                    null,
                    null
            )

            if ( addressCur != null ) {

                while (addressCur.moveToNext()) {
                    label         = addressCur.getString(addressCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredPostal.TYPE)) ?: ""
                    street          = addressCur.getString(addressCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredPostal.STREET)) ?: ""
                    city            = addressCur.getString(addressCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredPostal.CITY)) ?: ""
                    postalCode      = addressCur.getString(addressCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredPostal.POSTCODE)) ?: ""
                    state           = addressCur.getString(addressCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredPostal.REGION)) ?: ""
                    country         = addressCur.getString(addressCur.getColumnIndex(ContactsContract.CommonDataKinds.StructuredPostal.COUNTRY)) ?: ""

                    jsonAddress.put("label", label)
                    jsonAddress.put("street", street)
                    jsonAddress.put("city", city)
                    jsonAddress.put("postalcode", postalCode)
                    jsonAddress.put("state", state)
                    jsonAddress.put("country", country)
                    jsonCurrentContact.accumulate("address", jsonAddress)

                }

            }
            addressCur?.close()


            Log.e("", "jsonCurrentContact: $jsonCurrentContact")



            // PUT the current JSON object info into an array
            jsonAllContacts.accumulate("contacts", jsonCurrentContact)
        }
        cursor.close()

    }).start()

} else {
    cursor?.close()
}

marmor · Accepted Answer · 2019-09-26T19:48:27.443

Those 3000 queries you mentioned can be reduced to just one, and it should finish rather quickly.

We'll take advantage of two things when improving the code:

All the Data stored in the ContactsContract.CommonDataKinds.XXX tables is actually stored in a single big table called Data.
There's an implicit join in ContactsContract that allows us to select columns from ContactsContract.Contacts when querying over Data

To make the code simpler, I advise you to define a Contact object to store in memory the info we find for a single contact, and use a HashMap to map a contact-ID to a Contact object

Read more about this here: https://developer.android.com/reference/android/provider/ContactsContract.Data.html

Here's some code to get you started:

Map<Long, Contact> contacts = new HashMap<>();

// If you need item type / label, add Data.DATA2 & Data.DATA3 to the projection
String[] projection = {Data.CONTACT_ID, Data.DISPLAY_NAME, Data.MIMETYPE, Data.DATA1};
// Add more types to the selection if needed, e.g. StructuredName
String selection = Data.MIMETYPE + " IN ('" + Phone.CONTENT_ITEM_TYPE + "', '" + Email.CONTENT_ITEM_TYPE + "', '" + StructuredPostal.CONTENT_ITEM_TYPE + "')"; 
Cursor cur = cr.query(Data.CONTENT_URI, projection, selection, null, null);

// Loop through the data
while (cur.moveToNext()) {
    long id = cur.getLong(0);
    String name = cur.getString(1);
    String mime = cur.getString(2); // email / phone / postal
    String data = cur.getString(3); // the actual info, e.g. +1-212-555-1234

    // get the Contact class from the HashMap, or create a new one and add it to the Hash
    Contact contact;
    if (contacts.containsKey(id)) {
        contact = contacts.get(id);
    } else {
        contact = new Contact(id);
        contact.setDisplayName(name);
        // start with empty Sets for phones and emails
        // instead of HashSets you can use some object to retain more info about the data item (e.g. label)
        contact.setPhoneNumbers(new HashSet<>()); 
        contact.setEmails(new HashSet<>());
        contact.setAddresses(new HashSet<>());
        contacts.put(id, contact);
    } 

    switch (mime) {
        case Phone.CONTENT_ITEM_TYPE: 
            contact.getPhoneNumbers().add(data);
            break;
        case Email.CONTENT_ITEM_TYPE: 
            contact.getEmails().add(data);
            break;
        case StructuredPostal.CONTENT_ITEM_TYPE: 
            contact.getAddresses().add(data);
            break;
    }
}
cur.close();

FOLLOW UP Great progress on performance! Now here are some small tweaks to take your new code even further:

Try to avoid the sort, pass null for sort, and adjust your code to handle the data in whatever order it comes, SQLite sort can sometimes slow down queries significantly
Reduce your projection to only fields you actually need, the more stuff you put on your projection the bigger the data size that needs to be shifted around between processes on the device, which leads to more chunks with fewer rows in each
Don't getString on all fields in projection, if some fields are only used by the StructuredPostal read those only for StructuredPostal rows and not for each iteration.

report in the comments what were you able to achieve with the above tips...

I posted the `Kotlin` answer below. Thanks to your logic, it went from 3 minutes to 500ms to get the list of contacts and process it. — Mike Casan Ballester, Sep 26 '19 at 15:02
i think you can get even better performance with some tweaks, i'll post my comments above in my answer — marmor, Sep 26 '19 at 19:42

score 0 · Answer 2 · answered Sep 26 '19 at 15:00

Here is my Kotlin code based on the logic of @marmor answer.

This optimised code retrieves 1500 contacts in a S7 Samsung in 500ms instead of 3 minutes in the question code.

For the sake of completeness I left the piece of codes that aggregate the data into a JSON Object and all the details for running the process in a separate thread.

Thread(Runnable {
    val resolver: ContentResolver = contentResolver
    var jsonToSend = JSONObject()
    var jsonAllContacts = JSONObject()
    val jsonName = JSONObject()
    val jsonEmail = JSONObject()
    val jsonPhone = JSONObject()
    val jsonAddress = JSONObject()

    var cursorPosition = 0
    var currentProgress: Int

    val projection = arrayOf(
            ContactsContract.Data.CONTACT_ID,
            ContactsContract.Data.DISPLAY_NAME,
            ContactsContract.Data.MIMETYPE,
            ContactsContract.Data.DATA1,
            ContactsContract.Data.DATA2,
            ContactsContract.Data.DATA3,
            ContactsContract.Data.DATA4,
            ContactsContract.Data.DATA5,
            ContactsContract.Data.DATA6,
            ContactsContract.Data.DATA7,
            ContactsContract.Data.DATA8,
            ContactsContract.Data.DATA9,
            ContactsContract.Data.DATA10,
            ContactsContract.Data.DATA11,
            ContactsContract.Data.DATA12,
            ContactsContract.Data.DATA13,
            ContactsContract.Data.DATA14
    )
    val selection = ContactsContract.Data.MIMETYPE + " IN ('" + ContactsContract.CommonDataKinds.StructuredName.CONTENT_ITEM_TYPE + "', '" + ContactsContract.CommonDataKinds.Phone.CONTENT_ITEM_TYPE + "', '" + ContactsContract.CommonDataKinds.Email.CONTENT_ITEM_TYPE + "', '" + ContactsContract.CommonDataKinds.StructuredPostal.CONTENT_ITEM_TYPE + "')"

    val order = "CASE WHEN " + ContactsContract.Data.MIMETYPE + " = '" + ContactsContract.CommonDataKinds.StructuredName.CONTENT_ITEM_TYPE + "' THEN 0 ELSE 1 END ASC, '" + ContactsContract.Data.CONTACT_ID + "'"
    val cursor = resolver.query(
            ContactsContract.Data.CONTENT_URI,
            projection,
            selection,
            null,
            order
    )

    Log.e("cursor", "cursor STARTED")

    if (cursor != null) {
        while (cursor.moveToNext())
        {
            cursorPosition++
            currentProgress = ((cursorPosition.toFloat() / cursor.count.toFloat()) * 100).toInt()


            // SELECT the data needed from the standardized table
            val id = cursor.getLong(0).toString()
            val name = cursor.getString(1)
            val mime = cursor.getString(2) // email / phone / postal
            val data1 = cursor.getString(3) // the actual info, e.g. +1-212-555-1234
            val data2 = cursor.getString(4)
            val data3 = cursor.getString(5) 
            val data4 = cursor.getString(6) 
            val data5 = cursor.getString(7) 
            val data6 = cursor.getString(8) 
            val data7 = cursor.getString(9) 
            val data8 = cursor.getString(10) 
            val data9 = cursor.getString(11) 
            val data10 = cursor.getString(12) 
            val data11 = cursor.getString(13) 
            val data12 = cursor.getString(14) 
            val data13 = cursor.getString(15) 
            val data14 = cursor.getString(16) 
            // get the Contact class from the HashMap, or create a new one and add it to the Hash




            when (mime) {
                ContactsContract.CommonDataKinds.StructuredName.CONTENT_ITEM_TYPE -> {
                    /**
                     * Type     Alias           Data column
                     * String   DISPLAY_NAME    DATA1
                     * String   GIVEN_NAME  DATA2
                     * String   FAMILY_NAME DATA3
                     * String   PREFIX  DATA4   Common prefixes in English names are "Mr", "Ms", "Dr" etc.
                     * String   MIDDLE_NAME DATA5
                     * String   SUFFIX  DATA6   Common suffixes in English names are "Sr", "Jr", "III" etc.
                     * String   PHONETIC_GIVEN_NAME DATA7   Used for phonetic spelling of the name, e.g. Pinyin, Katakana, Hiragana
                     * String   PHONETIC_MIDDLE_NAME    DATA8
                     * String   PHONETIC_FAMILY_NAME    DATA9
                     */
                    //Log.e("contact", "Name-- name:$name -- id:$id == 1:$data1 // 2:$data2 // 3:$data3 // 4: $data4 // 5:$data5 // 6:$data6 // 7:$data7 // 8:$data8 // 9:$data9 // 10:$data10 // 11:$data11 // 12:$data12 // 13:$data13 // 14:$data14")

                    val currentJSON = JSONObject()
                    val fullName = if ( data5 != null && (data5 != data3) ) {
                        "$data5 $data3"
                    } else {
                        data3 ?: data1 // PUT data1 as last resort because a contact with no names inputted will return something else (e.g. email address)
                    }

                    currentJSON.put("given", data2)
                    currentJSON.put("family", fullName)


                    jsonName.put( id, currentJSON)



                }
                ContactsContract.CommonDataKinds.Email.CONTENT_ITEM_TYPE -> {
                    /**
                     * IMPROVE - check that valueForJSON / data1 is a properly formatted email. It sometimes saves in the email the name of the person instead of the email.
                     * e.g. "aze qsd" instead of "aze.qsd@gmail.com"
                     */
                    //Log.e("contact", "Email-- name:$name -- id:$id == 1:$data1 // 2:$data2 // 3:$data3 // 4: $data4 // 5:$data5 // 6:$data6 // 7:$data7 // 8:$data8 // 9:$data9 // 10:$data10 // 11:$data11 // 12:$data12 // 13:$data13 // 14:$data14")

                    val valueForJSON = data1


                    if ( jsonEmail.has(id) ) {
                        val indexString = jsonEmail[id].toString()
                        val indexArray = JSONObject(indexString)

                        if ( !indexArray.has(valueForJSON) ) {
                            jsonEmail.put( id, indexArray.put( valueForJSON, data2 ) )
                        }
                    } else {
                        jsonEmail.put( id, JSONObject().put( valueForJSON, data2 ) )
                    }



                }
                ContactsContract.CommonDataKinds.Phone.CONTENT_ITEM_TYPE -> {
                    /**
                     * Type Alias   Data column
                    String  NUMBER  ContactsContract.DataColumns.DATA1
                    int ContactsContract.CommonDataKinds.CommonColumns.TYPE ContactsContract.DataColumns.DATA2  Allowed values are:
                    ContactsContract.CommonDataKinds.BaseTypes.TYPE_CUSTOM. Put the actual type in ContactsContract.CommonDataKinds.CommonColumns.LABEL.
                    TYPE_HOME
                    TYPE_MOBILE
                    TYPE_WORK
                    etc..
                    String  ContactsContract.CommonDataKinds.CommonColumns.LABEL    ContactsContract.DataColumns.DATA3
                    String  ContactsContract.CommonDataKinds.CommonColumns.NORMALIZED_NUMBER    ContactsContract.DataColumns.DATA4

                     */
                    //Log.e("contact", "Phone-- name:$name -- id:$id == 1:$data1 // 2:$data2 // 3:$data3 // 4: $data4 // 5:$data5 // 6:$data6 // 7:$data7 // 8:$data8 // 9:$data9 // 10:$data10 // 11:$data11 // 12:$data12 // 13:$data13 // 14:$data14")

                    val valueForJSON = data4 ?: data1


                    if ( jsonPhone.has(id) ) {
                        val indexString = jsonPhone[id].toString()
                        val indexArray = JSONObject(indexString)

                        if ( !indexArray.has(valueForJSON) ) {
                            jsonPhone.put( id, indexArray.put( valueForJSON, data2 ) )
                        }
                    } else {
                        jsonPhone.put( id, JSONObject().put( valueForJSON, data2 ) )
                    }

                }
                ContactsContract.CommonDataKinds.StructuredPostal.CONTENT_ITEM_TYPE -> {
                    /**
                     * Type Alias   Data column
                    String  FORMATTED_ADDRESS   ContactsContract.DataColumns.DATA1
                    int ContactsContract.CommonDataKinds.CommonColumns.TYPE ContactsContract.DataColumns.DATA2  Allowed values are:
                    ContactsContract.CommonDataKinds.BaseTypes.TYPE_CUSTOM. Put the actual type in ContactsContract.CommonDataKinds.CommonColumns.LABEL.
                    TYPE_HOME
                    TYPE_WORK
                    TYPE_OTHER
                    String  ContactsContract.CommonDataKinds.CommonColumns.LABEL    ContactsContract.DataColumns.DATA3
                    String  STREET  ContactsContract.DataColumns.DATA4
                    String  POBOX   ContactsContract.DataColumns.DATA5  Post Office Box number
                    String  NEIGHBORHOOD    ContactsContract.DataColumns.DATA6
                    String  CITY    ContactsContract.DataColumns.DATA7
                    String  REGION  ContactsContract.DataColumns.DATA8
                    String  POSTCODE    ContactsContract.DataColumns.DATA9
                    String  COUNTRY ContactsContract.DataColumns.DATA10
                     */
                    //Log.e("contact", "Address-- name:$name -- id:$id == 1:$data1 // 2:$data2 // 3:$data3 // 4: $data4 // 5:$data5 // 6:$data6 // 7:$data7 // 8:$data8 // 9:$data9 // 10:$data10 // 11:$data11 // 12:$data12 // 13:$data13 // 14:$data14")


                    val currentJSON = JSONObject()
                    val valueForJSON = data1
                    currentJSON.put("street", data4)
                    currentJSON.put("city", data7)
                    currentJSON.put("postcode", data9)
                    currentJSON.put("state", data8)
                    currentJSON.put("country", data10)



                    if ( jsonAddress.has(id) ) {
                        val indexString = jsonAddress[id].toString()
                        val indexArray = JSONObject(indexString)

                        if ( !indexArray.has(valueForJSON) ) {
                            jsonAddress.put( id, indexArray.put( valueForJSON, currentJSON ) )
                        }
                    } else {
                        jsonAddress.put( id, JSONObject().put( valueForJSON, currentJSON ) )
                    }

                }
            }



            runOnUiThread {
                //PROGRESS HERE
                //myProgressBar?.visibility = View.VISIBLE
                myProgressBar?.progress = currentProgress
            }

        }

    }
    cursor?.close()



    // PUT some data to the first level of the nested JSON object
    jsonToSend.put("source", "2")



    /**
     * ADD the type of contact info to the main jsonAllContacts Object
     */
    jsonAllContacts = addNameToCurrentObject(jsonAllContacts, jsonName)
    jsonAllContacts = addNewKeyToCurrentObject(jsonAllContacts, jsonPhone, "email")
    jsonAllContacts = addNewKeyToCurrentObject(jsonAllContacts, jsonPhone, "phone")
    jsonAllContacts = addNewKeyToCurrentObject(jsonAllContacts, jsonAddress, "address")



    /**
     * Remove the IDs that was used for aggregation of data
     */
    jsonToSend = removeIDofObject(jsonToSend, jsonAllContacts)

}).start()

How to improve performance when query many data on ALL contacts from Android ContactsContract.Contacts?

OBJECTIVE

PROBLEM

CODE

2 Answers2