Changeset 18473 in josm


Ignore:
Timestamp:
2022-06-08T17:48:47+02:00 (23 months ago)
Author:
taylor.smock
Message:

Fix #22032: Various memory enhancements for MVT tiles

There is about a 88% decrease in memory allocations due to MVT tiles.

The majority of changes came from the following:

  1. Avoiding ArrayList#grow calls by using a ByteArrayOutputStream that gets passed around
  2. Caching nodes to avoid QuadBuckets#search calls

This additionally fixes a bug where VectorRelation#setMembers does not
work properly and adds a putAll method in Tagged (there is a default
method).

Location:
trunk
Files:
15 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java

    r18431 r18473  
    22package org.openstreetmap.josm.data.imagery.vectortile.mapbox;
    33
     4import java.io.ByteArrayOutputStream;
    45import java.io.IOException;
    56import java.text.NumberFormat;
    … …  
    2728    /**
    2829     * The number format instance to use (using a static instance gets rid of quite o few allocations)
    29      * Doing this reduced the allocations of {@link #parseTagValue(String, Layer, Number)} from 22.79% of parent to
     30     * Doing this reduced the allocations of {@link #parseTagValue(String, Layer, Number, List)} from 22.79% of parent to
    3031     * 12.2% of parent.
    3132     */
    3233    private static final NumberFormat NUMBER_FORMAT = NumberFormat.getNumberInstance(Locale.ROOT);
     34    private static final String[] EMPTY_STRING_ARRAY = new String[0];
    3335    /**
    3436     * The geometry of the feature. Required.
    … …  
    4850     * The tags of the feature. Optional.
    4951     */
    50     private TagMap tags;
     52    private final TagMap tags;
    5153    private Geometry geometryObject;
    5254
    … …  
    6264        GeometryTypes geometryTypeTemp = GeometryTypes.UNKNOWN;
    6365        String key = null;
     66        // Use a list where we can grow capacity easily (TagMap will do an array copy every time a tag is added)
     67        // This lets us avoid most array copies (i.e., this should only happen if some software decided it would be
     68        // a good idea to have multiple tag fields).
     69        // By avoiding array copies in TagMap, Feature#init goes from 339 MB to 188 MB.
     70        ArrayList<String> tagList = null;
    6471        try (ProtobufParser parser = new ProtobufParser(record.getBytes())) {
     72            ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(4);
    6573            while (parser.hasNext()) {
    66                 try (ProtobufRecord next = new ProtobufRecord(parser)) {
     74                try (ProtobufRecord next = new ProtobufRecord(byteArrayOutputStream, parser)) {
    6775                    if (next.getField() == TAG_FIELD) {
    68                         if (tags == null) {
    69                             tags = new TagMap();
     76                        // This is packed in v1 and v2
     77                        ProtobufPacked packed = new ProtobufPacked(byteArrayOutputStream, next.getBytes());
     78                        if (tagList == null) {
     79                            tagList = new ArrayList<>(packed.getArray().length);
     80                        } else {
     81                            tagList.ensureCapacity(tagList.size() + packed.getArray().length);
    7082                        }
    71                         // This is packed in v1 and v2
    72                         ProtobufPacked packed = new ProtobufPacked(next.getBytes());
    7383                        for (Number number : packed.getArray()) {
    74                             key = parseTagValue(key, layer, number);
     84                            key = parseTagValue(key, layer, number, tagList);
    7585                        }
    7686                    } else if (next.getField() == GEOMETRY_FIELD) {
    7787                        // This is packed in v1 and v2
    78                         ProtobufPacked packed = new ProtobufPacked(next.getBytes());
     88                        ProtobufPacked packed = new ProtobufPacked(byteArrayOutputStream, next.getBytes());
    7989                        CommandInteger currentCommand = null;
    8090                        for (Number number : packed.getArray()) {
    … …  
    91101                        // TODO fallback to non-packed
    92102                    } else if (next.getField() == GEOMETRY_TYPE_FIELD) {
    93                         geometryTypeTemp = GeometryTypes.values()[next.asUnsignedVarInt().intValue()];
     103                        // by using getAllValues, we avoid 12.4 MB allocations
     104                        geometryTypeTemp = GeometryTypes.getAllValues()[next.asUnsignedVarInt().intValue()];
    94105                    } else if (next.getField() == ID_FIELD) {
    95106                        tId = next.asUnsignedVarInt().longValue();
    … …  
    101112        this.geometryType = geometryTypeTemp;
    102113        record.close();
     114        if (tagList != null && !tagList.isEmpty()) {
     115            this.tags = new TagMap(tagList.toArray(EMPTY_STRING_ARRAY));
     116        } else {
     117            this.tags = null;
     118        }
    103119    }
    104120
    … …  
    109125     * @param layer  The layer with key/value information
    110126     * @param number The number to get the value from
     127     * @param tagList The list to add the new value to
    111128     * @return The new key (if {@code null}, then a value was parsed and added to tags)
    112129     */
    113     private String parseTagValue(String key, Layer layer, Number number) {
     130    private String parseTagValue(String key, Layer layer, Number number, List<String> tagList) {
    114131        if (key == null) {
    115132            key = layer.getKey(number.intValue());
    116133        } else {
     134            tagList.add(key);
    117135            Object value = layer.getValue(number.intValue());
    118136            if (value instanceof Double || value instanceof Float) {
    … …  
    122140                try {
    123141                    NUMBER_FORMAT.setGroupingUsed(false);
    124                     this.tags.put(key, NUMBER_FORMAT.format(value));
     142                    tagList.add(Utils.intern(NUMBER_FORMAT.format(value)));
    125143                } finally {
    126144                    NUMBER_FORMAT.setGroupingUsed(grouping);
    127145                }
    128146            } else {
    129                 this.tags.put(key, Utils.intern(value.toString()));
     147                tagList.add(Utils.intern(value.toString()));
    130148            }
    131149            key = null;

  • trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Geometry.java

    r18185 r18473  
    1919 */
    2020public class Geometry {
    21     final Collection<Shape> shapes = new ArrayList<>();
     21    final Collection<Shape> shapes;
    2222
    2323    /**
    … …  
    2929    public Geometry(GeometryTypes geometryType, List<CommandInteger> commands) {
    3030        if (geometryType == GeometryTypes.POINT) {
    31             for (CommandInteger command : commands) {
    32                 final short[] operations = command.getOperations();
    33                 // Each MoveTo command is a new point
    34                 if (command.getType() == Command.MoveTo && operations.length % 2 == 0 && operations.length > 0) {
    35                     for (int i = 0; i < operations.length / 2; i++) {
    36                         // Just using Ellipse2D since it extends Shape
    37                         shapes.add(new Ellipse2D.Float(operations[2 * i], operations[2 * i + 1], 0, 0));
    38                     }
    39                 } else {
    40                     throw new IllegalArgumentException(tr("{0} with {1} arguments is not understood", geometryType, operations.length));
     31            // This gets rid of most of the expensive array copies from ArrayList#grow
     32            shapes = new ArrayList<>(commands.size());
     33            initializePoints(geometryType, commands);
     34        } else if (geometryType == GeometryTypes.LINESTRING || geometryType == GeometryTypes.POLYGON) {
     35            // This gets rid of most of the expensive array copies from ArrayList#grow
     36            shapes = new ArrayList<>(1);
     37            initializeWayGeometry(geometryType, commands);
     38        } else {
     39            shapes = Collections.emptyList();
     40        }
     41    }
     42
     43    /**
     44     * Initialize point geometry
     45     * @param geometryType The geometry type (used for logging)
     46     * @param commands The commands to use to create the geometry
     47     */
     48    private void initializePoints(GeometryTypes geometryType, List<CommandInteger> commands) {
     49        for (CommandInteger command : commands) {
     50            final short[] operations = command.getOperations();
     51            // Each MoveTo command is a new point
     52            if (command.getType() == Command.MoveTo && operations.length % 2 == 0 && operations.length > 0) {
     53                for (int i = 0; i < operations.length / 2; i++) {
     54                    // Just using Ellipse2D since it extends Shape
     55                    shapes.add(new Ellipse2D.Float(operations[2 * i], operations[2 * i + 1], 0, 0));
    4156                }
     57            } else {
     58                throw new IllegalArgumentException(tr("{0} with {1} arguments is not understood", geometryType, operations.length));
    4259            }
    43         } else if (geometryType == GeometryTypes.LINESTRING || geometryType == GeometryTypes.POLYGON) {
    44             Path2D.Float line = null;
    45             Area area = null;
    46             // MVT uses delta encoding. Each feature starts at (0, 0).
    47             int x = 0;
    48             int y = 0;
    49             // Area is used to determine the inner/outer of a polygon
    50             final int maxArraySize = commands.stream().filter(command -> command.getType() != Command.ClosePath)
    51                     .mapToInt(command -> command.getOperations().length).sum();
    52             final List<Integer> xArray = new ArrayList<>(maxArraySize);
    53             final List<Integer> yArray = new ArrayList<>(maxArraySize);
    54             for (CommandInteger command : commands) {
    55                 final short[] operations = command.getOperations();
    56                 // Technically, there is no reason why there can be multiple MoveTo operations in one command, but that is undefined behavior
    57                 if (command.getType() == Command.MoveTo && operations.length == 2) {
    58                     x += operations[0];
    59                     y += operations[1];
    60                     line = new Path2D.Float();
    61                     line.moveTo(x, y);
     60        }
     61    }
     62
     63    /**
     64     * Initialize way geometry
     65     * @param geometryType The geometry type
     66     * @param commands The commands to use to create the geometry
     67     */
     68    private void initializeWayGeometry(GeometryTypes geometryType, List<CommandInteger> commands) {
     69        Path2D.Float line = null;
     70        Area area = null;
     71        // MVT uses delta encoding. Each feature starts at (0, 0).
     72        int x = 0;
     73        int y = 0;
     74        // Area is used to determine the inner/outer of a polygon
     75        final int maxArraySize = commands.stream().filter(command -> command.getType() != Command.ClosePath)
     76                .mapToInt(command -> command.getOperations().length).sum();
     77        final List<Integer> xArray = new ArrayList<>(maxArraySize);
     78        final List<Integer> yArray = new ArrayList<>(maxArraySize);
     79        for (CommandInteger command : commands) {
     80            final short[] operations = command.getOperations();
     81            // Technically, there is no reason why there can be multiple MoveTo operations in one command, but that is undefined behavior
     82            if (command.getType() == Command.MoveTo && operations.length == 2) {
     83                x += operations[0];
     84                y += operations[1];
     85                // Avoid fairly expensive Arrays.copyOf calls
     86                line = new Path2D.Float(Path2D.WIND_NON_ZERO, commands.size());
     87                line.moveTo(x, y);
     88                xArray.add(x);
     89                yArray.add(y);
     90                shapes.add(line);
     91            } else if (command.getType() == Command.LineTo && operations.length % 2 == 0 && line != null) {
     92                for (int i = 0; i < operations.length / 2; i++) {
     93                    x += operations[2 * i];
     94                    y += operations[2 * i + 1];
    6295                    xArray.add(x);
    6396                    yArray.add(y);
    64                     shapes.add(line);
    65                 } else if (command.getType() == Command.LineTo && operations.length % 2 == 0 && line != null) {
    66                     for (int i = 0; i < operations.length / 2; i++) {
    67                         x += operations[2 * i];
    68                         y += operations[2 * i + 1];
    69                         xArray.add(x);
    70                         yArray.add(y);
    71                         line.lineTo(x, y);
    72                     }
     97                    line.lineTo(x, y);
     98                }
    7399                // ClosePath should only be used with Polygon geometry
    74                 } else if (geometryType == GeometryTypes.POLYGON && command.getType() == Command.ClosePath && line != null) {
    75                     shapes.remove(line);
    76                     // new Area() closes the line if it isn't already closed
    77                     if (area == null) {
    78                         area = new Area();
    79                         shapes.add(area);
    80                     }
     100            } else if (geometryType == GeometryTypes.POLYGON && command.getType() == Command.ClosePath && line != null) {
     101                shapes.remove(line);
     102                // new Area() closes the line if it isn't already closed
     103                if (area == null) {
     104                    area = new Area();
     105                    shapes.add(area);
     106                }
    81107
    82                     final double areaAreaSq = calculateSurveyorsArea(xArray.stream().mapToInt(i -> i).toArray(),
    83                             yArray.stream().mapToInt(i -> i).toArray());
    84                     Area nArea = new Area(line);
    85                     // SonarLint thinks that this is never > 0. It can be.
    86                     if (areaAreaSq > 0) {
    87                         area.add(nArea);
    88                     } else if (areaAreaSq < 0) {
    89                         area.exclusiveOr(nArea);
    90                     } else {
    91                         throw new IllegalArgumentException(tr("{0} cannot have zero area", geometryType));
    92                     }
    93                     xArray.clear();
    94                     yArray.clear();
     108                final double areaAreaSq = calculateSurveyorsArea(xArray.stream().mapToInt(i -> i).toArray(),
     109                        yArray.stream().mapToInt(i -> i).toArray());
     110                Area nArea = new Area(line);
     111                // SonarLint thinks that this is never > 0. It can be.
     112                if (areaAreaSq > 0) {
     113                    area.add(nArea);
     114                } else if (areaAreaSq < 0) {
     115                    area.exclusiveOr(nArea);
    95116                } else {
    96                     throw new IllegalArgumentException(tr("{0} with {1} arguments is not understood", geometryType, operations.length));
     117                    throw new IllegalArgumentException(tr("{0} cannot have zero area", geometryType));
    97118                }
     119                xArray.clear();
     120                yArray.clear();
     121            } else {
     122                throw new IllegalArgumentException(tr("{0} with {1} arguments is not understood", geometryType, operations.length));
    98123            }
    99124        }

  • trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/GeometryTypes.java

    r17867 r18473  
    1919    POLYGON;
    2020
     21    private static final GeometryTypes[] CACHED_VALUES = values();
    2122    /**
    2223     * Rings used by {@link GeometryTypes#POLYGON}
    … …  
    2930        InteriorRing
    3031    }
     32
     33    /**
     34     * A replacement for {@link #values()} which can be used when there are no changes to the underlying array.
     35     * This is useful for avoiding unnecessary allocations.
     36     * @return A cached array from {@link #values()}. Do not modify.
     37     */
     38    static GeometryTypes[] getAllValues() {
     39        return CACHED_VALUES;
     40    }
    3141}

  • trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Layer.java

    r17867 r18473  
    11// License: GPL. For details, see LICENSE file.
    22package org.openstreetmap.josm.data.imagery.vectortile.mapbox;
     3
    34import static org.openstreetmap.josm.tools.I18n.tr;
    45
     6import java.io.ByteArrayOutputStream;
    57import java.io.IOException;
    68import java.util.ArrayList;
    … …  
    810import java.util.Collection;
    911import java.util.Collections;
    10 import java.util.HashSet;
     12import java.util.HashMap;
    1113import java.util.List;
    1214import java.util.Map;
    … …  
    1820import org.openstreetmap.josm.data.protobuf.ProtobufRecord;
    1921import org.openstreetmap.josm.tools.Destroyable;
    20 import org.openstreetmap.josm.tools.Logging;
    2122
    2223/**
    … …  
    100101    public Layer(Collection<ProtobufRecord> records) throws IOException {
    101102        // Do the unique required fields first
    102         Map<Integer, List<ProtobufRecord>> sorted = records.stream().collect(Collectors.groupingBy(ProtobufRecord::getField));
    103         this.version = sorted.getOrDefault((int) VERSION_FIELD, Collections.emptyList()).parallelStream()
    104           .map(ProtobufRecord::asUnsignedVarInt).map(Number::byteValue).findFirst().orElse(DEFAULT_VERSION);
    105         // Per spec, we cannot continue past this until we have checked the version number
    106         if (this.version != 1 && this.version != 2) {
    107             throw new IllegalArgumentException(tr("We do not understand version {0} of the vector tile specification", this.version));
    108         }
    109         this.name = sorted.getOrDefault((int) NAME_FIELD, Collections.emptyList()).parallelStream().map(ProtobufRecord::asString).findFirst()
    110                 .orElseThrow(() -> new IllegalArgumentException(tr("Vector tile layers must have a layer name")));
    111         this.extent = sorted.getOrDefault((int) EXTENT_FIELD, Collections.emptyList()).parallelStream().map(ProtobufRecord::asUnsignedVarInt)
    112                 .map(Number::intValue).findAny().orElse(DEFAULT_EXTENT);
    113 
    114         sorted.getOrDefault((int) KEY_FIELD, Collections.emptyList()).parallelStream().map(ProtobufRecord::asString)
    115                 .forEachOrdered(this.keyList::add);
    116         sorted.getOrDefault((int) VALUE_FIELD, Collections.emptyList()).parallelStream().map(ProtobufRecord::getBytes)
    117                 .map(ProtobufParser::new).map(parser1 -> {
    118                     try {
    119                         return new ProtobufRecord(parser1);
    120                     } catch (IOException e) {
    121                         Logging.error(e);
    122                         return null;
    123                     }
    124                 })
    125                 .filter(Objects::nonNull)
    126                 .map(value -> ValueFields.MAPPERS.parallelStream()
    127                         .filter(v -> v.getField() == value.getField())
    128                         .map(v -> v.convertValue(value)).findFirst()
    129                         .orElseThrow(() -> new IllegalArgumentException(tr("Unknown field in vector tile layer value ({0})", value.getField()))))
    130                 .forEachOrdered(this.valueList::add);
    131         Collection<IOException> exceptions = new HashSet<>(0);
    132         this.featureCollection = sorted.getOrDefault((int) FEATURE_FIELD, Collections.emptyList()).parallelStream().map(feature -> {
    133             try {
    134                 return new Feature(this, feature);
    135             } catch (IOException e) {
    136                 exceptions.add(e);
     103        Map<Integer, List<ProtobufRecord>> sorted = new HashMap<>(records.size());
     104        byte tVersion = DEFAULT_VERSION;
     105        String tName = null;
     106        int tExtent = DEFAULT_EXTENT;
     107        final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(4);
     108        for (ProtobufRecord protobufRecord : records) {
     109            if (protobufRecord.getField() == VERSION_FIELD) {
     110                tVersion = protobufRecord.asUnsignedVarInt().byteValue();
     111                // Per spec, we cannot continue past this until we have checked the version number
     112                if (tVersion != 1 && tVersion != 2) {
     113                    throw new IllegalArgumentException(tr("We do not understand version {0} of the vector tile specification", tVersion));
     114                }
     115            } else if (protobufRecord.getField() == NAME_FIELD) {
     116                tName = protobufRecord.asString();
     117            } else if (protobufRecord.getField() == EXTENT_FIELD) {
     118                tExtent = protobufRecord.asUnsignedVarInt().intValue();
     119            } else if (protobufRecord.getField() == KEY_FIELD) {
     120                this.keyList.add(protobufRecord.asString());
     121            } else if (protobufRecord.getField() == VALUE_FIELD) {
     122                parseValueRecord(byteArrayOutputStream, protobufRecord);
     123            } else {
     124                sorted.computeIfAbsent(protobufRecord.getField(), i -> new ArrayList<>(records.size())).add(protobufRecord);
    137125            }
    138             return null;
    139         }).collect(Collectors.toList());
    140         if (!exceptions.isEmpty()) {
    141             throw exceptions.iterator().next();
     126        }
     127        this.version = tVersion;
     128        if (tName == null) {
     129            throw new IllegalArgumentException(tr("Vector tile layers must have a layer name"));
     130        }
     131        this.name = tName;
     132        this.extent = tExtent;
     133
     134        this.featureCollection = new ArrayList<>(sorted.getOrDefault((int) FEATURE_FIELD, Collections.emptyList()).size());
     135        for (ProtobufRecord protobufRecord : sorted.getOrDefault((int) FEATURE_FIELD, Collections.emptyList())) {
     136            this.featureCollection.add(new Feature(this, protobufRecord));
    142137        }
    143138        // Cleanup bytes (for memory)
    144         for (ProtobufRecord record : records) {
    145             record.close();
     139        for (ProtobufRecord protobufRecord : records) {
     140            protobufRecord.close();
     141        }
     142    }
     143
     144    private void parseValueRecord(ByteArrayOutputStream byteArrayOutputStream, ProtobufRecord protobufRecord)
     145            throws IOException {
     146        try (ProtobufParser parser = new ProtobufParser(protobufRecord.getBytes())) {
     147            ProtobufRecord protobufRecord2 = new ProtobufRecord(byteArrayOutputStream, parser);
     148            int field = protobufRecord2.getField();
     149            int valueListSize = this.valueList.size();
     150            for (Layer.ValueFields<?> mapper : ValueFields.MAPPERS) {
     151                if (mapper.getField() == field) {
     152                    this.valueList.add(mapper.convertValue(protobufRecord2));
     153                    break;
     154                }
     155            }
     156            if (valueListSize == this.valueList.size()) {
     157                throw new IllegalArgumentException(tr("Unknown field in vector tile layer value ({0})", field));
     158            }
    146159        }
    147160    }

  • trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTTile.java

    r18156 r18473  
    99import java.util.List;
    1010import java.util.Objects;
    11 import java.util.stream.Collectors;
    1211
    1312import org.openstreetmap.gui.jmapviewer.Tile;
    … …  
    5453            ProtobufParser parser = new ProtobufParser(inputStream);
    5554            Collection<ProtobufRecord> protobufRecords = parser.allRecords();
    56             this.layers = new HashSet<>();
    57             this.layers = protobufRecords.stream().map(protoBufRecord -> {
    58                 Layer mvtLayer = null;
     55            this.layers = new HashSet<>(protobufRecords.size());
     56            for (ProtobufRecord protoBufRecord : protobufRecords) {
    5957                if (protoBufRecord.getField() == Layer.LAYER_FIELD) {
    6058                    try (ProtobufParser tParser = new ProtobufParser(protoBufRecord.getBytes())) {
    61                         mvtLayer = new Layer(tParser.allRecords());
     59                        this.layers.add(new Layer(tParser.allRecords()));
    6260                    } catch (IOException e) {
    6361                        Logging.error(e);
    … …  
    6765                    }
    6866                }
    69                 return mvtLayer;
    70             }).collect(Collectors.toCollection(HashSet::new));
     67            }
     68            this.layers = new HashSet<>(this.layers);
     69
    7170            this.extent = layers.stream().filter(Objects::nonNull).mapToInt(Layer::getExtent).max().orElse(Layer.DEFAULT_EXTENT);
    7271            if (this.getData() != null) {

  • trunk/src/org/openstreetmap/josm/data/osm/AbstractPrimitive.java

    r18208 r18473  
    66import java.text.MessageFormat;
    77import java.time.Instant;
     8import java.util.ArrayList;
    89import java.util.Arrays;
    910import java.util.Collection;
    … …  
    610611            keysChangedImpl(originalKeys);
    611612        }
     613    }
     614
     615    @Override
     616    public void putAll(Map<String, String> tags) {
     617        if (tags == null || tags.isEmpty()) {
     618            return;
     619        }
     620        // Defensive copy of keys
     621        String[] newKeys = keys;
     622        Map<String, String> originalKeys = getKeys();
     623        List<Map.Entry<String, String>> tagsToAdd = new ArrayList<>(tags.size());
     624        for (Map.Entry<String, String> tag : tags.entrySet()) {
     625            if (!Utils.isBlank(tag.getKey())) {
     626                int keyIndex = indexOfKey(newKeys, tag.getKey());
     627                // Realistically, we will not hit the newKeys == null branch. If it is null, keyIndex is always < 1
     628                if (keyIndex < 0 || newKeys == null) {
     629                    tagsToAdd.add(tag);
     630                } else {
     631                    newKeys[keyIndex + 1] = tag.getValue();
     632                }
     633            }
     634        }
     635        if (!tagsToAdd.isEmpty()) {
     636            int index = newKeys != null ? newKeys.length : 0;
     637            newKeys = newKeys != null ? Arrays.copyOf(newKeys, newKeys.length + 2 * tagsToAdd.size()) : new String[2 * tagsToAdd.size()];
     638            for (Map.Entry<String, String> tag : tagsToAdd) {
     639                newKeys[index++] = tag.getKey();
     640                newKeys[index++] = tag.getValue();
     641            }
     642            keys = newKeys;
     643        }
     644        keysChangedImpl(originalKeys);
    612645    }
    613646

  • trunk/src/org/openstreetmap/josm/data/osm/Tagged.java

    r18211 r18473  
    7070
    7171    /**
     72     * Add all key/value pairs. This <i>may</i> be more performant than {@link #put}, depending upon the implementation.
     73     * By default, this calls {@link #put} for each map entry.
     74     * @param tags The tag map to add
     75     * @since xxx
     76     */
     77    default void putAll(Map<String, String> tags) {
     78        for (Map.Entry<String, String> entry : tags.entrySet()) {
     79            put(entry.getKey(), entry.getValue());
     80        }
     81    }
     82
     83    /**
    7284     * Replies the value of the given key; null, if there is no value for this key
    7385     *

  • trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufPacked.java

    r18431 r18473  
    1313 */
    1414public class ProtobufPacked {
     15    private static final Number[] NO_NUMBERS = new Number[0];
    1516    private final byte[] bytes;
    1617    private final Number[] numbers;
    … …  
    2021     * Create a new ProtobufPacked object
    2122     *
     23     * @param byteArrayOutputStream A reusable ByteArrayOutputStream (helps to reduce memory allocations)
    2224     * @param bytes The packed bytes
    2325     */
    24     public ProtobufPacked(byte[] bytes) {
     26    public ProtobufPacked(ByteArrayOutputStream byteArrayOutputStream, byte[] bytes) {
    2527        this.location = 0;
    2628        this.bytes = bytes;
    27         List<Number> numbersT = new ArrayList<>();
     29
     30        // By creating a list of size bytes.length, we avoid 36 MB of allocations from list growth. This initialization
     31        // only adds 3.7 MB to the ArrayList#init calls. Note that the real-world test case (Mapillary vector tiles)
     32        // primarily created Shorts.
     33        List<Number> numbersT = new ArrayList<>(bytes.length);
    2834        // By reusing a ByteArrayOutputStream, we can reduce allocations in nextVarInt from 230 MB to 74 MB.
    29         ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(4);
    3035        while (this.location < bytes.length) {
    3136            numbersT.add(ProtobufParser.convertByteArray(this.nextVarInt(byteArrayOutputStream), ProtobufParser.VAR_INT_BYTE_SIZE));
    32             byteArrayOutputStream.reset();
    3337        }
    3438
    35         this.numbers = new Number[numbersT.size()];
    36         for (int i = 0; i < numbersT.size(); i++) {
    37             this.numbers[i] = numbersT.get(i);
    38         }
     39        this.numbers = numbersT.toArray(NO_NUMBERS);
    3940    }
    4041
    … …  
    5152        // In a real world test, the largest List<Byte> seen had 3 elements. Use 4 to avoid most new array allocations.
    5253        // Memory allocations went from 368 MB to 280 MB by using an initial array allocation. When using a
    53         // ByteArrayOutputStream, it went down to 230 MB.
     54        // ByteArrayOutputStream, it went down to 230 MB. By further reusing the ByteArrayOutputStream between method
     55        // calls, it went down further to 73 MB.
    5456        while ((this.bytes[this.location] & ProtobufParser.MOST_SIGNIFICANT_BYTE)
    5557          == ProtobufParser.MOST_SIGNIFICANT_BYTE) {
    … …  
    5961        // The last byte doesn't drop the most significant bit
    6062        byteArrayOutputStream.write(this.bytes[this.location++]);
    61         return byteArrayOutputStream.toByteArray();
     63        try {
     64            return byteArrayOutputStream.toByteArray();
     65        } finally {
     66            byteArrayOutputStream.reset();
     67        }
    6268    }
    6369}

  • trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufParser.java

    r18431 r18473  
    122122    public Collection<ProtobufRecord> allRecords() throws IOException {
    123123        Collection<ProtobufRecord> records = new ArrayList<>();
     124        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(4);
    124125        while (this.hasNext()) {
    125             records.add(new ProtobufRecord(this));
     126            records.add(new ProtobufRecord(byteArrayOutputStream, this));
    126127        }
    127128        return records;
    … …  
    197198     * Get the next delimited message ({@link WireType#LENGTH_DELIMITED})
    198199     *
     200     * @param byteArrayOutputStream A reusable stream to write bytes to. This can significantly reduce the allocations
     201     *                              (150 MB to 95 MB in a test area).
    199202     * @return The next length delimited message
    200203     * @throws IOException - if an IO error occurs
    201204     */
    202     public byte[] nextLengthDelimited() throws IOException {
    203         int length = convertByteArray(this.nextVarInt(), VAR_INT_BYTE_SIZE).intValue();
     205    public byte[] nextLengthDelimited(ByteArrayOutputStream byteArrayOutputStream) throws IOException {
     206        int length = convertByteArray(this.nextVarInt(byteArrayOutputStream), VAR_INT_BYTE_SIZE).intValue();
    204207        return readNextBytes(length);
    205208    }
    … …  
    208211     * Get the next var int ({@code WireType#VARINT})
    209212     *
     213     * @param byteArrayOutputStream A reusable stream to write bytes to. This can significantly reduce the allocations
     214     *                              (150 MB to 95 MB in a test area).
    210215     * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum})
    211216     * @throws IOException - if an IO error occurs
    212217     */
    213     public byte[] nextVarInt() throws IOException {
     218    public byte[] nextVarInt(ByteArrayOutputStream byteArrayOutputStream) throws IOException {
    214219        // Using this reduces the allocations from 150 MB to 95 MB.
    215         final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(4);
    216220        int currentByte = this.nextByte();
    217221        while ((byte) (currentByte & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE && currentByte > 0) {
    … …  
    222226        // The last byte doesn't drop the most significant bit
    223227        byteArrayOutputStream.write(currentByte);
    224         return byteArrayOutputStream.toByteArray();
     228        try {
     229            return byteArrayOutputStream.toByteArray();
     230        } finally {
     231            byteArrayOutputStream.reset();
     232        }
    225233    }
    226234

  • trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufRecord.java

    r18432 r18473  
    22package org.openstreetmap.josm.data.protobuf;
    33
     4import java.io.ByteArrayOutputStream;
    45import java.io.IOException;
    56import java.nio.charset.StandardCharsets;
    … …  
    2223     * Create a new Protobuf record
    2324     *
     25     * @param byteArrayOutputStream A reusable ByteArrayOutputStream to avoid unnecessary allocations
    2426     * @param parser The parser to use to create the record
    2527     * @throws IOException - if an IO error occurs
    2628     */
    27     public ProtobufRecord(ProtobufParser parser) throws IOException {
    28         Number number = ProtobufParser.convertByteArray(parser.nextVarInt(), ProtobufParser.VAR_INT_BYTE_SIZE);
     29    public ProtobufRecord(ByteArrayOutputStream byteArrayOutputStream, ProtobufParser parser) throws IOException {
     30        Number number = ProtobufParser.convertByteArray(parser.nextVarInt(byteArrayOutputStream), ProtobufParser.VAR_INT_BYTE_SIZE);
    2931        // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3}
    3032        this.field = (int) number.longValue() >> 3;
    … …  
    4345
    4446        if (this.type == WireType.VARINT) {
    45             this.bytes = parser.nextVarInt();
     47            this.bytes = parser.nextVarInt(byteArrayOutputStream);
    4648        } else if (this.type == WireType.SIXTY_FOUR_BIT) {
    4749            this.bytes = parser.nextFixed64();
    … …  
    4951            this.bytes = parser.nextFixed32();
    5052        } else if (this.type == WireType.LENGTH_DELIMITED) {
    51             this.bytes = parser.nextLengthDelimited();
     53            this.bytes = parser.nextLengthDelimited(byteArrayOutputStream);
    5254        } else {
    5355            this.bytes = EMPTY_BYTES;

  • trunk/src/org/openstreetmap/josm/data/vector/VectorDataStore.java

    r17994 r18473  
    1313import java.util.Collection;
    1414import java.util.Collections;
     15import java.util.HashMap;
    1516import java.util.List;
     17import java.util.Map;
    1618import java.util.Objects;
    1719
    18 import org.openstreetmap.gui.jmapviewer.Coordinate;
    1920import org.openstreetmap.gui.jmapviewer.Tile;
    2021import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
    2122import org.openstreetmap.josm.data.IQuadBucketType;
     23import org.openstreetmap.josm.data.coor.ILatLon;
     24import org.openstreetmap.josm.data.coor.LatLon;
    2225import org.openstreetmap.josm.data.imagery.vectortile.VectorTile;
    2326import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Feature;
    … …  
    3639import org.openstreetmap.josm.tools.JosmRuntimeException;
    3740import org.openstreetmap.josm.tools.Logging;
     41import org.openstreetmap.josm.tools.Utils;
    3842
    3943/**
    … …  
    157161
    158162    private synchronized <T extends Tile & VectorTile> VectorNode pointToNode(T tile, Layer layer,
    159       Collection<VectorPrimitive> featureObjects, int x, int y) {
     163      Collection<VectorPrimitive> featureObjects, int x, int y, final Map<ILatLon, VectorNode> nodeMap) {
    160164        final BBox tileBbox;
    161165        if (tile instanceof IQuadBucketType) {
    … …  
    169173        }
    170174        final int layerExtent = layer.getExtent();
    171         final ICoordinate coords = new Coordinate(
     175        final LatLon coords = new LatLon(
    172176                tileBbox.getMaxLat() - (tileBbox.getMaxLat() - tileBbox.getMinLat()) * y / layerExtent,
    173177                tileBbox.getMinLon() + (tileBbox.getMaxLon() - tileBbox.getMinLon()) * x / layerExtent
    174178        );
     179        if (nodeMap.containsKey(coords)) {
     180            return nodeMap.get(coords);
     181        }
    175182        final Collection<VectorNode> nodes = this.store
    176           .searchNodes(new BBox(coords.getLon(), coords.getLat(), VectorDataSet.DUPE_NODE_DISTANCE));
     183          .searchNodes(new BBox(coords.lon(), coords.lat(), VectorDataSet.DUPE_NODE_DISTANCE));
    177184        final VectorNode node;
    178185        if (!nodes.isEmpty()) {
    … …  
    203210        node.setCoor(coords);
    204211        featureObjects.add(node);
     212        nodeMap.put(node.getCoor(), node);
    205213        return node;
    206214    }
    207215
    208216    private <T extends Tile & VectorTile> List<VectorWay> pathToWay(T tile, Layer layer,
    209       Collection<VectorPrimitive> featureObjects, Path2D shape) {
     217      Collection<VectorPrimitive> featureObjects, Path2D shape, Map<ILatLon, VectorNode> nodeMap) {
    210218        final PathIterator pathIterator = shape.getPathIterator(null);
    211         final List<VectorWay> ways = pathIteratorToObjects(tile, layer, featureObjects, pathIterator).stream()
    212           .filter(VectorWay.class::isInstance).map(VectorWay.class::cast).collect(toList());
     219        final List<VectorWay> ways = new ArrayList<>(
     220                Utils.filteredCollection(pathIteratorToObjects(tile, layer, featureObjects, pathIterator, nodeMap), VectorWay.class));
    213221        // These nodes technically do not exist, so we shouldn't show them
    214         ways.stream().flatMap(way -> way.getNodes().stream())
    215           .filter(prim -> !prim.isTagged() && prim.getReferrers(true).size() == 1 && prim.getId() <= 0)
    216           .forEach(prim -> {
    217               prim.setDisabled(true);
    218               prim.setVisible(false);
    219           });
     222        for (VectorWay way : ways) {
     223            for (VectorNode node : way.getNodes()) {
     224                if (!node.hasKeys() && node.getReferrers(true).size() == 1 && node.getId() <= 0) {
     225                    node.setDisabled(true);
     226                    node.setVisible(false);
     227                }
     228            }
     229        }
    220230        return ways;
    221231    }
    222232
    223233    private <T extends Tile & VectorTile> List<VectorPrimitive> pathIteratorToObjects(T tile, Layer layer,
    224       Collection<VectorPrimitive> featureObjects, PathIterator pathIterator) {
     234      Collection<VectorPrimitive> featureObjects, PathIterator pathIterator, Map<ILatLon, VectorNode> nodeMap) {
    225235        final List<VectorNode> nodes = new ArrayList<>();
    226236        final double[] coords = new double[6];
    … …  
    243253            }
    244254            if (PathIterator.SEG_MOVETO == type || PathIterator.SEG_LINETO == type) {
    245                 final VectorNode node = pointToNode(tile, layer, featureObjects, (int) coords[0], (int) coords[1]);
     255                final VectorNode node = pointToNode(tile, layer, featureObjects, (int) coords[0], (int) coords[1], nodeMap);
    246256                nodes.add(node);
    247257            } else if (PathIterator.SEG_CLOSE != type) {
    … …  
    260270
    261271    private <T extends Tile & VectorTile> VectorRelation areaToRelation(T tile, Layer layer,
    262       Collection<VectorPrimitive> featureObjects, Area area) {
     272      Collection<VectorPrimitive> featureObjects, Area area, Map<ILatLon, VectorNode> nodeMap) {
    263273        VectorRelation vectorRelation = new VectorRelation(layer.getName());
    264         for (VectorPrimitive member : pathIteratorToObjects(tile, layer, featureObjects, area.getPathIterator(null))) {
     274        for (VectorPrimitive member : pathIteratorToObjects(tile, layer, featureObjects, area.getPathIterator(null), nodeMap)) {
    265275            final String role;
    266276            if (member instanceof VectorWay && ((VectorWay) member).isClosed()) {
    … …  
    280290     */
    281291    public <T extends Tile & VectorTile> void addDataTile(T tile) {
     292        // Using a map reduces the cost of addFeatureData from 2,715,158,632 bytes to 235,042,184 bytes (-91.3%)
     293        // This was somewhat variant, with some runs being closer to ~560 MB (still -80%).
     294        final Map<ILatLon, VectorNode> nodeMap = new HashMap<>();
    282295        for (Layer layer : tile.getLayers()) {
    283296            for (Feature feature : layer.getFeatures()) {
    284297                try {
    285                     addFeatureData(tile, layer, feature);
     298                    addFeatureData(tile, layer, feature, nodeMap);
    286299                } catch (IllegalArgumentException e) {
    287300                    Logging.error("Cannot add vector data for feature {0} of tile {1}: {2}", feature, tile, e.getMessage());
    … …  
    299312    }
    300313
    301     private <T extends Tile & VectorTile> void addFeatureData(T tile, Layer layer, Feature feature) {
    302         List<VectorPrimitive> featureObjects = new ArrayList<>();
    303         List<VectorPrimitive> primaryFeatureObjects = feature.getGeometryObject().getShapes().stream()
    304                 .map(shape -> shapeToPrimaryFeatureObject(tile, layer, shape, featureObjects)).collect(toList());
     314    private <T extends Tile & VectorTile> void addFeatureData(T tile, Layer layer, Feature feature, Map<ILatLon, VectorNode> nodeMap) {
     315        // This will typically be larger than primaryFeatureObjects, but this at least avoids quite a few ArrayList#grow calls
     316        List<VectorPrimitive> featureObjects = new ArrayList<>(feature.getGeometryObject().getShapes().size());
     317        List<VectorPrimitive> primaryFeatureObjects = new ArrayList<>(featureObjects.size());
     318        for (Shape shape : feature.getGeometryObject().getShapes()) {
     319            primaryFeatureObjects.add(shapeToPrimaryFeatureObject(tile, layer, shape, featureObjects, nodeMap));
     320        }
    305321        final VectorPrimitive primitive;
    306322        if (primaryFeatureObjects.size() == 1) {
    … …  
    311327        } else if (!primaryFeatureObjects.isEmpty()) {
    312328            VectorRelation relation = new VectorRelation(layer.getName());
    313             primaryFeatureObjects.stream().map(prim -> new VectorRelationMember("", prim))
    314               .forEach(relation::addRelationMember);
     329            List<VectorRelationMember> members = new ArrayList<>(primaryFeatureObjects.size());
     330            for (VectorPrimitive prim : primaryFeatureObjects) {
     331                members.add(new VectorRelationMember("", prim));
     332            }
     333            relation.setMembers(members);
    315334            primitive = relation;
    316335        } else {
    … …  
    326345        }
    327346        if (feature.getTags() != null) {
    328             feature.getTags().forEach(primitive::put);
    329         }
    330         featureObjects.forEach(this::addPrimitive);
    331         primaryFeatureObjects.forEach(this::addPrimitive);
     347            primitive.putAll(feature.getTags());
     348        }
     349        for (VectorPrimitive prim : featureObjects) {
     350            this.addPrimitive(prim);
     351        }
     352        for (VectorPrimitive prim : primaryFeatureObjects) {
     353            this.addPrimitive(prim);
     354        }
    332355        try {
    333356            this.addPrimitive(primitive);
    … …  
    339362
    340363    private <T extends Tile & VectorTile> VectorPrimitive shapeToPrimaryFeatureObject(
    341             T tile, Layer layer, Shape shape, List<VectorPrimitive> featureObjects) {
     364            T tile, Layer layer, Shape shape, List<VectorPrimitive> featureObjects, Map<ILatLon, VectorNode> nodeMap) {
    342365        final VectorPrimitive primitive;
    343366        if (shape instanceof Ellipse2D) {
    344367            primitive = pointToNode(tile, layer, featureObjects,
    345                     (int) ((Ellipse2D) shape).getCenterX(), (int) ((Ellipse2D) shape).getCenterY());
     368                    (int) ((Ellipse2D) shape).getCenterX(), (int) ((Ellipse2D) shape).getCenterY(), nodeMap);
    346369        } else if (shape instanceof Path2D) {
    347             primitive = pathToWay(tile, layer, featureObjects, (Path2D) shape).stream().findFirst().orElse(null);
     370            primitive = pathToWay(tile, layer, featureObjects, (Path2D) shape, nodeMap).stream().findFirst().orElse(null);
    348371        } else if (shape instanceof Area) {
    349             primitive = areaToRelation(tile, layer, featureObjects, (Area) shape);
     372            primitive = areaToRelation(tile, layer, featureObjects, (Area) shape, nodeMap);
    350373            primitive.put(RELATION_TYPE, MULTIPOLYGON_TYPE);
    351374        } else {

  • trunk/src/org/openstreetmap/josm/data/vector/VectorPrimitive.java

    r17867 r18473  
    33
    44import java.util.Arrays;
     5import java.util.Collections;
    56import java.util.List;
    67import java.util.Map;
    … …  
    118119    @Override
    119120    public final List<VectorPrimitive> getReferrers(boolean allowWithoutDataset) {
     121        if (this.referrers == null) {
     122            return Collections.emptyList();
     123        } else if (this.referrers instanceof VectorPrimitive) {
     124            return Collections.singletonList((VectorPrimitive) this.referrers);
     125        }
    120126        return referrers(allowWithoutDataset, VectorPrimitive.class)
    121127          .collect(Collectors.toList());

  • trunk/src/org/openstreetmap/josm/data/vector/VectorRelation.java

    r17867 r18473  
    9191        this.members.clear();
    9292        this.members.addAll(members);
     93        for (VectorRelationMember member : members) {
     94            member.getMember().addReferrer(this);
     95        }
     96        cachedBBox = null;
    9397    }
    9498

  • trunk/test/unit/org/openstreetmap/josm/data/protobuf/ProtobufRecordTest.java

    r17862 r18473  
    44import static org.junit.jupiter.api.Assertions.assertEquals;
    55
    6 
     6import java.io.ByteArrayOutputStream;
    77import java.io.IOException;
    88
    … …  
    1616    void testFixed32() throws IOException {
    1717        ProtobufParser parser = new ProtobufParser(ProtobufTest.toByteArray(new int[] {0x0d, 0x00, 0x00, 0x80, 0x3f}));
    18         ProtobufRecord thirtyTwoBit = new ProtobufRecord(parser);
     18        ProtobufRecord thirtyTwoBit = new ProtobufRecord(new ByteArrayOutputStream(), parser);
    1919        assertEquals(WireType.THIRTY_TWO_BIT, thirtyTwoBit.getType());
    2020        assertEquals(1f, thirtyTwoBit.asFloat());
    … …  
    2424    void testUnknown() throws IOException {
    2525        ProtobufParser parser = new ProtobufParser(ProtobufTest.toByteArray(new int[] {0x0f, 0x00, 0x00, 0x80, 0x3f}));
    26         ProtobufRecord unknown = new ProtobufRecord(parser);
     26        ProtobufRecord unknown = new ProtobufRecord(new ByteArrayOutputStream(), parser);
    2727        assertEquals(WireType.UNKNOWN, unknown.getType());
    2828        assertEquals(0, unknown.getBytes().length);

  • trunk/test/unit/org/openstreetmap/josm/data/protobuf/ProtobufTest.java

    r18037 r18473  
    88import java.awt.geom.Ellipse2D;
    99import java.io.ByteArrayInputStream;
     10import java.io.ByteArrayOutputStream;
    1011import java.io.File;
    1112import java.io.IOException;
    … …  
    172173    void testSimpleMessage() throws IOException {
    173174        ProtobufParser parser = new ProtobufParser(new byte[] {(byte) 0x08, (byte) 0x96, (byte) 0x01});
    174         ProtobufRecord record = new ProtobufRecord(parser);
     175        ProtobufRecord record = new ProtobufRecord(new ByteArrayOutputStream(), parser);
    175176        assertEquals(WireType.VARINT, record.getType());
    176177        assertEquals(150, record.asUnsignedVarInt().intValue());
Note: See TracChangeset for help on using the changeset viewer.