Mail has a cool way of making unread messages stand out. It is really easy to do this type of thing with NSTableView/NSOutlineView. Subclass the one you want, and override drawRow. Toss in the code you see below, and it should give a cool highlight on all expandable rows in an NSOutlineView (you will have to modify it to have it work in NSTableView — just remove the call to isExpandable).

- (void)drawRow:(int)row clipRect:(NSRect)clipRect {
    if (([self isExpandable:[self itemAtRow:row]]) && (![self isRowSelected:row])) {
        // Draw a light-blue “mail like” border around the row, if not selected
        NSRect rect = [self rectOfRow:row];
        [[[NSColor blueColor] colorWithAlphaComponent:50/255.0] set];
        NSBezierPath *path = [NSBezierPath bezierPath];
        [path setLineCapStyle:NSRoundLineCapStyle];
        [path setLineWidth:rect.size.height - 3];
        int rowLevel = [self levelForRow:row];
        float x = rect.origin.x + 10.0 + (rowLevel * [self indentationPerLevel]);
        float y = rect.origin.y + (rect.size.height / 2.0);
        [path moveToPoint:NSMakePoint(x,y)];
        [path lineToPoint:NSMakePoint(x + rect.size.width - 2*10.0, y)];
        [path stroke];
    }
    [super drawRow:row clipRect:clipRect];
}

NOTE: use this at your own risk. I think resizing the thing doesn’t quite work.