Java - Not getting right items using array and Streams to parse a file

Question

Am trying to parse the /etc/group file on a macOS Mojave 10.14.3 operating system using Java 1.8's stream feature.

The full set of lines inside my /etc/group file are as follows:

nobody:*:-2:
nogroup:*:-1:
wheel:*:0:root
daemon:*:1:root
kmem:*:2:root
sys:*:3:root
tty:*:4:root
operator:*:5:root
mail:*:6:_teamsserver
bin:*:7:
procview:*:8:root
procmod:*:9:root
owner:*:10:
everyone:*:12:
_taskgated:*:13:_taskgated
group:*:16:
staff:*:20:root
_networkd:*:24:
_installassistant:*:25:
_lp:*:26:
_postfix:*:27:
_postdrop:*:28:
certusers:*:29:root,_jabber,_postfix,_cyrus,_calendar,_dovecot
_keytabusers:*:30:_calendar,_jabber,_postfix
_scsd:*:31:
_ces:*:32:
_appstore:*:33:_appstore
utmp:*:45:
authedusers:*:50:
interactusers:*:51:
netusers:*:52:
consoleusers:*:53:
_mcxalr:*:54:
_appleevents:*:55:
_geod:*:56:
_devdocs:*:59:
_sandbox:*:60:
localaccounts:*:61:
netaccounts:*:62:
_mdnsresponder:*:65:
_uucp:*:66:
_ard:*:67:
dialer:*:68:
network:*:69:
_www:*:70:_devicemgr,_teamsserver
_eppc:*:71:_eppc
_cvs:*:72:
_svn:*:73:
_mysql:*:74:
_sshd:*:75:
_qtss:*:76:
_mailman:*:78:
_appserverusr:*:79:
admin:*:80:root
_appserveradm:*:81:
_clamav:*:82:
_amavisd:*:83:
_jabber:*:84:
_appowner:*:87:
_windowserver:*:88:
_spotlight:*:89:
accessibility:*:90:
_tokend:*:91:
_securityagent:*:92:
_calendar:*:93:_teamsserver
_teamsserver:*:94:_devicemgr
_update_sharing:*:95:
_installer:*:96:
_atsserver:*:97:
_lpadmin:*:98:
_unknown:*:99:
_lpoperator:*:100:
_softwareupdate:*:200:_softwareupdate
_guest:*:201:
_coreaudiod:*:202:
_screensaver:*:203:
_developer:*:204:
_locationd:*:205:
_detachedsig:*:207:_locationd
_trustevaluationagent:*:208:
_odchpass:*:209:_teamsserver
_timezone:*:210:
_lda:*:211:
_cvms:*:212:
_usbmuxd:*:213:
_postgres:*:216:_devicemgr,_calendar,_teamsserver,_xserverdocs
_devicemgr:*:220:
_webauthserver:*:221:_teamsserver,_devicemgr
_netbios:*:222:
_warmd:*:224:_warmd
_dovenull:*:227:
_netstatistics:*:228:
_assetcache:*:235:
_coremediaiod:*:236:
_launchservicesd:*:239:
_iconservices:*:240:
_distnote:*:241:
_nsurlsessiond:*:242:
_nsurlstoraged:*:243:
_displaypolicyd:*:244:
_astris:*:245:
_gamecontrollerd:*:247:
_mbsetupuser:*:248:
_ondemand:*:249:
_analyticsusers:*:250:_analyticsd,_networkd,_timed,_reportmemoryexception
_xserverdocs:*:251:
_wwwproxy:*:252:
_mobileasset:*:253:
_findmydevice:*:254:
_datadetectors:*:257:
_captiveagent:*:258:
_ctkd:*:259:
_applepay:*:260:
_hidd:*:261:
_cmiodalassistants:*:262:
_analyticsd:*:263:_analyticsd
_webdeveloper:*:264:
_fpsd:*:265:_fpsd
_timed:*:266:
_reportmemoryexception:*:269:_reportmemoryexception
com.apple.access_ftp:*:395:
com.apple.access_disabled:*:396:
com.apple.access_sessionkey:*:397:
com.apple.access_screensharing:*:398:
com.apple.access_ssh:*:399:

Group.java:

public class Group {
    private String gid;
    private String name;
    private String members;

    public Group(String line) {
        String[] items = line.split(":");

        if (items.length <= 3) {
            this.name = items[0];
            System.out.print("Name: " + name + ", ");
            this.gid = items[2];
            System.out.print(" gid: " + gid + ", ");
        }
        else if (items.length >=3 || this.name != null || "".equals(this.name)){
            this.members = items[3];
            System.out.println("Members: " + members);
        }
    }

    // Omitted getters & setters for brevity

}

Created this GroupsParser:

public class GroupParser {

    public static List<Group> getAllGroups(String line) {
        List<Group> groups = null;
        try (Stream<String> stream = Files.lines(Paths.get(line))) {

            groups = stream.filter(s -> s.charAt(0) != '#').map(Group::new)
                     .collect(Collectors.toCollection(ArrayList::new));
        } 
        catch (Exception e) {
            e.printStackTrace();
        }
        return groups;
    }

    public static void main(String[] args) {
        List<Group> groups = GroupParser.getAllGroups("/etc/group");
        for (Group group : groups) {
            System.out.println("Group: " + group.getName() + ", gid: " + group.getGid() + ", members: " + group.getMembers());
        }
    }
}

Received the following output (which is obviously out of order and missing some items):

Group: nobody, gid: -2, members: null
Group: nogroup, gid: -1, members: null
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: _teamsserver
Group: bin, gid: 7, members: null
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: owner, gid: 10, members: null
Group: everyone, gid: 12, members: null
Group: null, gid: null, members: _taskgated
Group: group, gid: 16, members: null
Group: null, gid: null, members: root
Group: _networkd, gid: 24, members: null
Group: _installassistant, gid: 25, members: null
Group: _lp, gid: 26, members: null
Group: _postfix, gid: 27, members: null
Group: _postdrop, gid: 28, members: null
Group: null, gid: null, members: root,_jabber,_postfix,_cyrus,_calendar,_dovecot
Group: null, gid: null, members: _calendar,_jabber,_postfix
Group: _scsd, gid: 31, members: null
Group: _ces, gid: 32, members: null
Group: null, gid: null, members: _appstore
Group: utmp, gid: 45, members: null
Group: authedusers, gid: 50, members: null
Group: interactusers, gid: 51, members: null
Group: netusers, gid: 52, members: null
Group: consoleusers, gid: 53, members: null
Group: _mcxalr, gid: 54, members: null
Group: _appleevents, gid: 55, members: null
Group: _geod, gid: 56, members: null
Group: _devdocs, gid: 59, members: null
Group: _sandbox, gid: 60, members: null
Group: localaccounts, gid: 61, members: null
Group: netaccounts, gid: 62, members: null
Group: _mdnsresponder, gid: 65, members: null
Group: _uucp, gid: 66, members: null
Group: _ard, gid: 67, members: null
Group: dialer, gid: 68, members: null
Group: network, gid: 69, members: null
Group: null, gid: null, members: _devicemgr,_teamsserver
Group: null, gid: null, members: _eppc
Group: _cvs, gid: 72, members: null
Group: _svn, gid: 73, members: null
Group: _mysql, gid: 74, members: null
Group: _sshd, gid: 75, members: null
Group: _qtss, gid: 76, members: null
Group: _mailman, gid: 78, members: null
Group: _appserverusr, gid: 79, members: null
Group: null, gid: null, members: root
Group: _appserveradm, gid: 81, members: null
Group: _clamav, gid: 82, members: null
Group: _amavisd, gid: 83, members: null
Group: _jabber, gid: 84, members: null
Group: _appowner, gid: 87, members: null
Group: _windowserver, gid: 88, members: null
Group: _spotlight, gid: 89, members: null
Group: accessibility, gid: 90, members: null
Group: _tokend, gid: 91, members: null
Group: _securityagent, gid: 92, members: null
Group: null, gid: null, members: _teamsserver
Group: null, gid: null, members: _devicemgr
Group: _update_sharing, gid: 95, members: null
Group: _installer, gid: 96, members: null
Group: _atsserver, gid: 97, members: null
Group: _lpadmin, gid: 98, members: null
Group: _unknown, gid: 99, members: null
Group: _lpoperator, gid: 100, members: null
Group: null, gid: null, members: _softwareupdate
Group: _guest, gid: 201, members: null
Group: _coreaudiod, gid: 202, members: null
Group: _screensaver, gid: 203, members: null
Group: _developer, gid: 204, members: null
Group: _locationd, gid: 205, members: null
Group: null, gid: null, members: _locationd
Group: _trustevaluationagent, gid: 208, members: null
Group: null, gid: null, members: _teamsserver
Group: _timezone, gid: 210, members: null
Group: _lda, gid: 211, members: null
Group: _cvms, gid: 212, members: null
Group: _usbmuxd, gid: 213, members: null
Group: null, gid: null, members: _devicemgr,_calendar,_teamsserver,_xserverdocs
Group: _devicemgr, gid: 220, members: null
Group: null, gid: null, members: _teamsserver,_devicemgr
Group: _netbios, gid: 222, members: null
Group: null, gid: null, members: _warmd
Group: _dovenull, gid: 227, members: null
Group: _netstatistics, gid: 228, members: null
Group: _assetcache, gid: 235, members: null
Group: _coremediaiod, gid: 236, members: null
Group: _launchservicesd, gid: 239, members: null
Group: _iconservices, gid: 240, members: null
Group: _distnote, gid: 241, members: null
Group: _nsurlsessiond, gid: 242, members: null
Group: _nsurlstoraged, gid: 243, members: null
Group: _displaypolicyd, gid: 244, members: null
Group: _astris, gid: 245, members: null
Group: _gamecontrollerd, gid: 247, members: null
Group: _mbsetupuser, gid: 248, members: null
Group: _ondemand, gid: 249, members: null
Group: null, gid: null, members: _analyticsd,_networkd,_timed,_reportmemoryexception
Group: _xserverdocs, gid: 251, members: null
Group: _wwwproxy, gid: 252, members: null
Group: _mobileasset, gid: 253, members: null
Group: _findmydevice, gid: 254, members: null
Group: _datadetectors, gid: 257, members: null
Group: _captiveagent, gid: 258, members: null
Group: _ctkd, gid: 259, members: null
Group: _applepay, gid: 260, members: null
Group: _hidd, gid: 261, members: null
Group: _cmiodalassistants, gid: 262, members: null
Group: null, gid: null, members: _analyticsd
Group: _webdeveloper, gid: 264, members: null
Group: null, gid: null, members: _fpsd
Group: _timed, gid: 266, members: null
Group: null, gid: null, members: _reportmemoryexception
Group: com.apple.access_ftp, gid: 395, members: null
Group: com.apple.access_disabled, gid: 396, members: null
Group: com.apple.access_sessionkey, gid: 397, members: null
Group: com.apple.access_screensharing, gid: 398, members: null
Group: com.apple.access_ssh, gid: 399, members: null
Group: nobody, gid: -2, members: null
Group: nogroup, gid: -1, members: null
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: null, gid: null, members: _teamsserver
Group: bin, gid: 7, members: null
Group: null, gid: null, members: root
Group: null, gid: null, members: root
Group: owner, gid: 10, members: null
Group: everyone, gid: 12, members: null
Group: null, gid: null, members: _taskgated
Group: group, gid: 16, members: null
Group: null, gid: null, members: root
Group: _networkd, gid: 24, members: null
Group: _installassistant, gid: 25, members: null
Group: _lp, gid: 26, members: null
Group: _postfix, gid: 27, members: null
Group: _postdrop, gid: 28, members: null
Group: null, gid: null, members: root,_jabber,_postfix,_cyrus,_calendar,_dovecot
Group: null, gid: null, members: _calendar,_jabber,_postfix
Group: _scsd, gid: 31, members: null
Group: _ces, gid: 32, members: null
Group: null, gid: null, members: _appstore
Group: utmp, gid: 45, members: null
Group: authedusers, gid: 50, members: null
Group: interactusers, gid: 51, members: null
Group: netusers, gid: 52, members: null
Group: consoleusers, gid: 53, members: null
Group: _mcxalr, gid: 54, members: null
Group: _appleevents, gid: 55, members: null
Group: _geod, gid: 56, members: null
Group: _devdocs, gid: 59, members: null
Group: _sandbox, gid: 60, members: null
Group: localaccounts, gid: 61, members: null
Group: netaccounts, gid: 62, members: null
Group: _mdnsresponder, gid: 65, members: null
Group: _uucp, gid: 66, members: null
Group: _ard, gid: 67, members: null
Group: dialer, gid: 68, members: null
Group: network, gid: 69, members: null
Group: null, gid: null, members: _devicemgr,_teamsserver
Group: null, gid: null, members: _eppc
Group: _cvs, gid: 72, members: null
Group: _svn, gid: 73, members: null
Group: _mysql, gid: 74, members: null
Group: _sshd, gid: 75, members: null
Group: _qtss, gid: 76, members: null
Group: _mailman, gid: 78, members: null
Group: _appserverusr, gid: 79, members: null
Group: null, gid: null, members: root
Group: _appserveradm, gid: 81, members: null
Group: _clamav, gid: 82, members: null
Group: _amavisd, gid: 83, members: null
Group: _jabber, gid: 84, members: null
Group: _appowner, gid: 87, members: null
Group: _windowserver, gid: 88, members: null
Group: _spotlight, gid: 89, members: null
Group: accessibility, gid: 90, members: null
Group: _tokend, gid: 91, members: null
Group: _securityagent, gid: 92, members: null
Group: null, gid: null, members: _teamsserver
Group: null, gid: null, members: _devicemgr
Group: _update_sharing, gid: 95, members: null
Group: _installer, gid: 96, members: null
Group: _atsserver, gid: 97, members: null
Group: _lpadmin, gid: 98, members: null
Group: _unknown, gid: 99, members: null
Group: _lpoperator, gid: 100, members: null
Group: null, gid: null, members: _softwareupdate
Group: _guest, gid: 201, members: null
Group: _coreaudiod, gid: 202, members: null
Group: _screensaver, gid: 203, members: null
Group: _developer, gid: 204, members: null
Group: _locationd, gid: 205, members: null
Group: null, gid: null, members: _locationd
Group: _trustevaluationagent, gid: 208, members: null
Group: null, gid: null, members: _teamsserver
Group: _timezone, gid: 210, members: null
Group: _lda, gid: 211, members: null
Group: _cvms, gid: 212, members: null
Group: _usbmuxd, gid: 213, members: null
Group: null, gid: null, members: _devicemgr,_calendar,_teamsserver,_xserverdocs
Group: _devicemgr, gid: 220, members: null
Group: null, gid: null, members: _teamsserver,_devicemgr
Group: _netbios, gid: 222, members: null
Group: null, gid: null, members: _warmd
Group: _dovenull, gid: 227, members: null
Group: _netstatistics, gid: 228, members: null
Group: _assetcache, gid: 235, members: null
Group: _coremediaiod, gid: 236, members: null
Group: _launchservicesd, gid: 239, members: null
Group: _iconservices, gid: 240, members: null
Group: _distnote, gid: 241, members: null
Group: _nsurlsessiond, gid: 242, members: null
Group: _nsurlstoraged, gid: 243, members: null
Group: _displaypolicyd, gid: 244, members: null
Group: _astris, gid: 245, members: null
Group: _gamecontrollerd, gid: 247, members: null
Group: _mbsetupuser, gid: 248, members: null
Group: _ondemand, gid: 249, members: null
Group: null, gid: null, members: _analyticsd,_networkd,_timed,_reportmemoryexception
Group: _xserverdocs, gid: 251, members: null
Group: _wwwproxy, gid: 252, members: null
Group: _mobileasset, gid: 253, members: null
Group: _findmydevice, gid: 254, members: null
Group: _datadetectors, gid: 257, members: null
Group: _captiveagent, gid: 258, members: null
Group: _ctkd, gid: 259, members: null
Group: _applepay, gid: 260, members: null
Group: _hidd, gid: 261, members: null
Group: _cmiodalassistants, gid: 262, members: null
Group: null, gid: null, members: _analyticsd
Group: _webdeveloper, gid: 264, members: null
Group: null, gid: null, members: _fpsd
Group: _timed, gid: 266, members: null
Group: null, gid: null, members: _reportmemoryexception
Group: com.apple.access_ftp, gid: 395, members: null
Group: com.apple.access_disabled, gid: 396, members: null
Group: com.apple.access_sessionkey, gid: 397, members: null
Group: com.apple.access_screensharing, gid: 398, members: null
Group: com.apple.access_ssh, gid: 399, members: null

Question(s):

1. This output doesn't seem ordered

   Group: nobody, gid: -2, members: null

   Group: nogroup, gid: -1, members: null

Why am I getting this, in between nogroup and wheel?

2. Is there a better way to extract these items? Why can't I see kmem or wheel anywhere?

3. Is there a way to check / fix this via Streams?

Answer 1

The core of the issue lies in the constructor you are using in mapping as .map(Group::new). The logic of the constructor implemented currently is a reason for a few questions arising. Trying to explain how below:

public Group(String line) {
    String[] items = line.split(":"); // you split up the string into contents based on ':' separator

in the condition to check if the contents are split into less than or equal to 3 parts, you've also made an assumption here that the size of array would at least be 3, which is what you intend to access when doing items[2]. Notice though, if the items.length could have been 2, the if condition would have successfully allowed you to end up getting an AIOOBE for this kind of access still.

    if (items.length <= 3) {
        this.name = items[0];
        System.out.print("Name: " + name + ", ");
        this.gid = items[2];
        System.out.print(" gid: " + gid + ", ");
    }

when the content of the string could be split into more than 3 parts, you tend not to initialize the name, gid for that Group and hence you've got those null s in your output.

    else {

        this.members = items[3];
        System.out.println("Members: " + members);
    }
}

---

Is there a better way to extract these items? Why can't I see kmem or wheel anywhere?

The section related to handling null values by initializing them would have solved for it.

---

To avoid NPE as well, you might want to initialize groups to an empty list;

List<Group> groups = new ArrayList<>();

---

A simpler implementation based on an assumption that the string would always have 3 or more components when split using : could be:

public Group(String line) {
    String[] items = line.split(":");
    this.name = items[0];
    this.gid = Integer.parseInt(items[2]);
    if (items.length > 3){
        this.members = items[3];
    }
}

Answer 2

Not sure if I'm understanding your question correctly. You just want to parse the file line by line, without any groupBy/merge action? items.length > 3 is unnecessary with Splitter from Google Guava or my library abacus-common. Here is the simple sample solution:

Splitter splitter = Splitter.with(':').trim(true);
Splitter memSplitter = Splitter.with(',').trim(true).omitEmptyStrings(true);

ExceptionalStream.lines(new File("./tmp.txt")) // Or StreamEx.of(IOUtil.readLines(new File("./tmp.txt")))
    .filter(s -> s.charAt(0) != '#')
    .map(s -> splitter.splitToArray(s))
    // .map(a -> Tuple.of(a[0], a[2], memSplitter.splitToArray(a[3])))
    // TODO whatever you need/want.
    .forEach(group -> N.println(group));

Answer 3

From my observations:

1. This output doesn't seem ordered

Change the code ArrayList::new to LinkedList::new to maintain ordering.

2. Is there a better way to extract these items? Why can't I see kmem or wheel anywhere?

The code if (items.length <= 3) { is the culprit. The else case does not assign values for fields other than members

3.Is there a way to check / fix this via Streams?

I think the fixes are done. Better ways - there will always.

Answer 4

Here below is the code which would help you :

class Group {

  private String gid;

  private String name;

  private String members;

  public Group(String line) {
    String[] consolidated = line.split(":");
    if (consolidated != null && consolidated.length > 0) {
      this.name = doesIndexExists(consolidated, 0) ? consolidated[0] : "";
      this.gid = doesIndexExists(consolidated, 2) ? consolidated[2] : "";
      this.members = consolidated.length < 4 ? "" : doesIndexExists(consolidated, 3) ? consolidated[3] : "";
    }
  }

  public String getGid() {
    return gid;
  }

  public void setGid(String gid) {
    this.gid = gid;
  }

  public String getName() {
    return name;
  }

  public String getMembers() {
    return members;
  }

  public boolean doesIndexExists(String[] inputArray, int index) {
    return inputArray.length > index && inputArray[index] != null;
  }

  public void setName(String name) {
    this.name = name;
  }

  public void setMembers(String members) {
    this.members = members;
  }

  @Override
  public String toString() {
    return "Group [gid=" + gid + ", name=" + name + ", members=" + members + "]";
  }

}

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

public class GroupParser {

  public static final String FILE_PATH = "YOURFILEPATH";

  public static boolean isNotEmpty(String inputContent) {
    return (inputContent != null && !"".equalsIgnoreCase(inputContent));
  }

  public static List<Group> getAllGroups(String line) throws IOException {
    final List<Group> groupList = new ArrayList<>();
    Files.readAllLines(Paths.get(line)).stream()
        .filter(lineInstance -> isNotEmpty(lineInstance) && !lineInstance.startsWith("#")).collect(Collectors.toList())
        .forEach(parsedInputObject -> groupList.add(new Group(parsedInputObject)));
    return groupList;
  }

  public static void main(String[] args) throws IOException {
    List<Group> groups = GroupParser.getAllGroups(FILE_PATH);
    for (Group group : groups) {
      System.out.println("Group: " + group.getName() + ", gid: " + group.getGid() + ", members: " + group.getMembers());
    }
  }
}

Answer 5

May I suggest a simpler fix than checking length of array or bringing in extra libraries: just add a few of the separators to guarantee sufficient number of array items:

String[] items = (line + "::::").split(":");

And String.split(String, int) could then save some negligible amount of time by not parsing more than the columns you want, if you want [up]tight code.

Well, that is all.