source: josm/trunk/src/org/openstreetmap/josm/data/osm/DataSetMerger.java@ 19078

Last change on this file since 19078 was 19078, checked in by taylor.smock, 4 months ago

Fix #4142: Track fully downloaded objects (patch by stoecker, GerdP, and myself)

The serialization move from PrimitiveData to AbstractPrimitive should be
reverted prior to 24.05 (see #23677).

The serialization move was required since we want to ensure that all downstream
users of AbstractPrimitive were not using the flags field, which was done by
making the field private instead of protected. They may still be using that
field (via updateFlags) which would not be caught by compile-time or runtime
errors.

Additionally, a good chunk of common functionality was moved up from
OsmPrimitive, even though much of it wasn't useful for PrimitiveData.
  • Property svn:eol-style set to native
File size: 21.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.osm;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.awt.geom.Area;
7import java.util.ArrayList;
8import java.util.Collection;
9import java.util.HashMap;
10import java.util.HashSet;
11import java.util.Iterator;
12import java.util.LinkedList;
13import java.util.List;
14import java.util.Map;
15import java.util.Set;
16
17import org.openstreetmap.josm.data.DataSource;
18import org.openstreetmap.josm.data.conflict.Conflict;
19import org.openstreetmap.josm.data.conflict.ConflictCollection;
20import org.openstreetmap.josm.gui.progress.ProgressMonitor;
21import org.openstreetmap.josm.tools.CheckParameterUtil;
22import org.openstreetmap.josm.tools.JosmRuntimeException;
23
24/**
25 * A dataset merger which takes a target and a source dataset and merges the source data set
26 * onto the target dataset.
27 *
28 */
29public class DataSetMerger {
30
31 /** the collection of conflicts created during merging */
32 private final ConflictCollection conflicts;
33
34 /** the target dataset for merging */
35 private final DataSet targetDataSet;
36 /** the source dataset where primitives are merged from */
37 private final DataSet sourceDataSet;
38
39 /**
40 * A map of all primitives that got replaced with other primitives.
41 * Key is the PrimitiveId in their dataset, the value is the PrimitiveId in my dataset
42 */
43 private final Map<PrimitiveId, PrimitiveId> mergedMap;
44 /** a set of primitive ids for which we have to fix references (to nodes and
45 * to relation members) after the first phase of merging
46 */
47 private final Set<PrimitiveId> objectsWithChildrenToMerge;
48 private final Set<OsmPrimitive> objectsToDelete;
49
50 /**
51 * constructor
52 *
53 * The visitor will merge <code>sourceDataSet</code> onto <code>targetDataSet</code>
54 *
55 * @param targetDataSet dataset with my primitives. Must not be null.
56 * @param sourceDataSet dataset with their primitives. Ignored, if null.
57 * @throws IllegalArgumentException if myDataSet is null
58 */
59 public DataSetMerger(DataSet targetDataSet, DataSet sourceDataSet) {
60 CheckParameterUtil.ensureParameterNotNull(targetDataSet, "targetDataSet");
61 this.targetDataSet = targetDataSet;
62 this.sourceDataSet = sourceDataSet;
63 conflicts = new ConflictCollection();
64 mergedMap = new HashMap<>();
65 objectsWithChildrenToMerge = new HashSet<>();
66 objectsToDelete = new HashSet<>();
67 }
68
69 /**
70 * Merges a primitive onto primitives dataset.
71 *
72 * If other.id != 0 it tries to merge it with an corresponding primitive from
73 * my dataset with the same id. If this is not possible a conflict is remembered
74 * in {@link #conflicts}.
75 *
76 * If other.id == 0 (new primitive) it tries to find a primitive in my dataset with id == 0 which
77 * is semantically equal. If it finds one it merges its technical attributes onto
78 * my primitive.
79 *
80 * @param source the primitive to merge
81 * @param candidates a set of possible candidates for a new primitive
82 */
83 protected void mergePrimitive(OsmPrimitive source, Collection<? extends OsmPrimitive> candidates) {
84 if (!source.isNew()) {
85 // try to merge onto a matching primitive with the same defined id
86 //
87 if (mergeById(source))
88 return;
89 } else {
90 // ignore deleted primitives from source
91 if (source.isDeleted()) return;
92
93 // try to merge onto a primitive which has no id assigned
94 // yet but which is equal in its semantic attributes
95 //
96 for (OsmPrimitive target : candidates) {
97 if (!target.isNew() || target.isDeleted()) {
98 continue;
99 }
100 if (target.hasEqualSemanticAttributes(source)) {
101 mergedMap.put(source.getPrimitiveId(), target.getPrimitiveId());
102 // copy the technical attributes from other version
103 target.setVisible(source.isVisible());
104 target.setUser(source.getUser());
105 target.setRawTimestamp(source.getRawTimestamp());
106 target.setModified(source.isModified());
107 objectsWithChildrenToMerge.add(source.getPrimitiveId());
108 return;
109 }
110 }
111 }
112
113 // If we get here we didn't find a suitable primitive in
114 // the target dataset. Create a clone and add it to the target dataset.
115 //
116 OsmPrimitive target;
117 switch (source.getType()) {
118 case NODE: target = source.isNew() ? new Node() : new Node(source.getId()); break;
119 case WAY: target = source.isNew() ? new Way() : new Way(source.getId()); break;
120 case RELATION: target = source.isNew() ? new Relation() : new Relation(source.getId()); break;
121 default: throw new AssertionError();
122 }
123 target.mergeFrom(source);
124 targetDataSet.addPrimitive(target);
125 mergedMap.put(source.getPrimitiveId(), target.getPrimitiveId());
126 objectsWithChildrenToMerge.add(source.getPrimitiveId());
127 }
128
129 protected OsmPrimitive getMergeTarget(OsmPrimitive mergeSource) {
130 PrimitiveId targetId = mergedMap.get(mergeSource.getPrimitiveId());
131 if (targetId == null)
132 return null;
133 return targetDataSet.getPrimitiveById(targetId);
134 }
135
136 protected void addConflict(Conflict<?> c) {
137 c.setMergedMap(mergedMap);
138 conflicts.add(c);
139 }
140
141 protected void addConflict(OsmPrimitive my, OsmPrimitive their) {
142 addConflict(new Conflict<>(my, their));
143 }
144
145 protected void fixIncomplete(Way other) {
146 Way myWay = (Way) getMergeTarget(other);
147 if (myWay == null)
148 throw new JosmRuntimeException(tr("Missing merge target for way with id {0}", other.getUniqueId()));
149 }
150
151 /**
152 * Postprocess the dataset and fix all merged references to point to the actual data.
153 */
154 public void fixReferences() {
155 for (Way w : sourceDataSet.getWays()) {
156 if (!conflicts.hasConflictForTheir(w) && objectsWithChildrenToMerge.contains(w.getPrimitiveId())) {
157 mergeNodeList(w);
158 fixIncomplete(w);
159 }
160 }
161 for (Relation r : sourceDataSet.getRelations()) {
162 if (!conflicts.hasConflictForTheir(r) && objectsWithChildrenToMerge.contains(r.getPrimitiveId())) {
163 mergeRelationMembers(r);
164 }
165 }
166
167 deleteMarkedObjects();
168 }
169
170 /**
171 * Deleted objects in objectsToDelete set and create conflicts for objects that cannot
172 * be deleted because they're referenced in the target dataset.
173 */
174 protected void deleteMarkedObjects() {
175 boolean flag;
176 do {
177 flag = false;
178 for (Iterator<OsmPrimitive> it = objectsToDelete.iterator(); it.hasNext();) {
179 OsmPrimitive target = it.next();
180 OsmPrimitive source = sourceDataSet.getPrimitiveById(target.getPrimitiveId());
181 if (source == null)
182 throw new JosmRuntimeException(
183 tr("Object of type {0} with id {1} was marked to be deleted, but it''s missing in the source dataset",
184 target.getType(), target.getUniqueId()));
185
186 List<OsmPrimitive> referrers = target.getReferrers();
187 if (referrers.isEmpty()) {
188 IPrimitive.resetPrimitiveChildren(target);
189 target.mergeFrom(source);
190 target.setDeleted(true);
191 it.remove();
192 flag = true;
193 } else {
194 for (OsmPrimitive referrer : referrers) {
195 // If one of object referrers isn't going to be deleted,
196 // add a conflict and don't delete the object
197 if (!objectsToDelete.contains(referrer)) {
198 addConflict(target, source);
199 it.remove();
200 flag = true;
201 break;
202 }
203 }
204 }
205
206 }
207 } while (flag);
208
209 if (!objectsToDelete.isEmpty()) {
210 // There are some more objects rest in the objectsToDelete set
211 // This can be because of cross-referenced relations.
212 for (OsmPrimitive osm: objectsToDelete) {
213 IPrimitive.resetPrimitiveChildren(osm);
214 }
215 for (OsmPrimitive osm: objectsToDelete) {
216 osm.setDeleted(true);
217 osm.mergeFrom(sourceDataSet.getPrimitiveById(osm.getPrimitiveId()));
218 }
219 }
220 }
221
222 /**
223 * Merges the node list of a source way onto its target way.
224 *
225 * @param source the source way
226 * @throws IllegalStateException if no target way can be found for the source way
227 * @throws IllegalStateException if there isn't a target node for one of the nodes in the source way
228 *
229 */
230 private void mergeNodeList(Way source) {
231 Way target = (Way) getMergeTarget(source);
232 if (target == null)
233 throw new IllegalStateException(tr("Missing merge target for way with id {0}", source.getUniqueId()));
234
235 List<Node> newNodes = new ArrayList<>(source.getNodesCount());
236 for (Node sourceNode : source.getNodes()) {
237 Node targetNode = (Node) getMergeTarget(sourceNode);
238 if (targetNode != null) {
239 newNodes.add(targetNode);
240 if (targetNode.isDeleted() && !conflicts.hasConflictForMy(targetNode)) {
241 addConflict(new Conflict<OsmPrimitive>(targetNode, sourceNode, true));
242 targetNode.setDeleted(false);
243 }
244 } else
245 throw new IllegalStateException(tr("Missing merge target for node with id {0}", sourceNode.getUniqueId()));
246 }
247 target.setNodes(newNodes);
248 }
249
250 /**
251 * Merges the relation members of a source relation onto the corresponding target relation.
252 * @param source the source relation
253 * @throws IllegalStateException if there is no corresponding target relation
254 * @throws IllegalStateException if there isn't a corresponding target object for one of the relation
255 * members in source
256 */
257 private void mergeRelationMembers(Relation source) {
258 Relation target = (Relation) getMergeTarget(source);
259 if (target == null)
260 throw new IllegalStateException(tr("Missing merge target for relation with id {0}", source.getUniqueId()));
261 List<RelationMember> newMembers = new LinkedList<>();
262 for (RelationMember sourceMember : source.getMembers()) {
263 OsmPrimitive targetMember = getMergeTarget(sourceMember.getMember());
264 if (targetMember == null)
265 throw new IllegalStateException(tr("Missing merge target of type {0} with id {1}",
266 sourceMember.getType(), sourceMember.getUniqueId()));
267 newMembers.add(new RelationMember(sourceMember.getRole(), targetMember));
268 if (targetMember.isDeleted() && !conflicts.hasConflictForMy(targetMember)) {
269 addConflict(new Conflict<>(targetMember, sourceMember.getMember(), true));
270 targetMember.setDeleted(false);
271 }
272 }
273 target.setMembers(newMembers);
274 }
275
276 /**
277 * Tries to merge a primitive <code>source</code> into an existing primitive with the same id.
278 *
279 * @param source the source primitive which is to be merged into a target primitive
280 * @return true, if this method was able to merge <code>source</code> into a target object; false, otherwise
281 */
282 private boolean mergeById(OsmPrimitive source) {
283 OsmPrimitive target = targetDataSet.getPrimitiveById(source.getId(), source.getType());
284 // merge other into an existing primitive with the same id, if possible
285 //
286 if (target == null)
287 return false;
288 // found a corresponding target, remember it
289 mergedMap.put(source.getPrimitiveId(), target.getPrimitiveId());
290
291 if (target.getVersion() > source.getVersion())
292 // target.version > source.version => keep target version
293 return true;
294
295 boolean mergeFromSource = false;
296 boolean haveSameVersion = target.getVersion() == source.getVersion();
297
298 if (haveSameVersion && !target.isModified() && !source.isModified()
299 && target.isVisible() != source.isVisible()) {
300 // Same version, but different "visible" attribute and neither of them are modified.
301 // It indicates a serious problem in datasets.
302 // For example, datasets can be fetched from different OSM servers or badly hand-modified.
303 // We shouldn't merge that datasets.
304 throw new DataIntegrityProblemException(tr("Conflict in ''visible'' attribute for object of type {0} with id {1}",
305 target.getType(), target.getId()));
306 }
307
308 if (!target.isModified() && source.isDeleted()) {
309 // target not modified and source is deleted
310 // So mark it to be deleted. See #20091
311 //
312 objectsToDelete.add(target);
313 } else if (source.isIncomplete()) {
314 // source is incomplete. Nothing to do.
315 //
316 } else if (target.isIncomplete()) {
317 // target is incomplete, source completes it
318 // => merge source into target
319 //
320 mergeFromSource = true;
321 } else if (target.isDeleted() && source.isDeleted() && !haveSameVersion) {
322 // both deleted. Source is newer. Take source. See #19783
323 mergeFromSource = true;
324 } else if (target.isDeleted() && !source.isDeleted() && haveSameVersion) {
325 // same version, but target is deleted. Assume target takes precedence
326 // otherwise too many conflicts when refreshing from the server
327 // but, if source is modified, there is a conflict
328 if (source.isModified()) {
329 addConflict(new Conflict<>(target, source, true));
330 }
331 // or, if source has a referrer that is not in the target dataset there is a conflict
332 // If target dataset refers to the deleted primitive, conflict will be added in fixReferences method
333 for (OsmPrimitive referrer: source.getReferrers()) {
334 if (targetDataSet.getPrimitiveById(referrer.getPrimitiveId()) == null) {
335 addConflict(new Conflict<>(target, source, true));
336 target.setDeleted(false);
337 break;
338 }
339 }
340 } else if (!target.isModified() && source.isModified()) {
341 // target not modified. We can assume that source is the most recent version.
342 // clone it into target.
343 mergeFromSource = true;
344 } else if (!target.isModified() && !source.isModified()) {
345 // both not modified. Merge nevertheless, even if versions are the same
346 // This helps when updating "empty" relations, see #4295
347 mergeFromSource = true;
348 } else if (target.isModified() && !source.isModified() && haveSameVersion) {
349 // target is same as source but target is modified
350 // => keep target and reset modified flag if target and source are semantically equal
351 if (target.hasEqualSemanticAttributes(source, false)) {
352 target.setModified(false);
353 }
354 } else if (source.isDeleted() != target.isDeleted()) {
355 // target is modified and deleted state differs.
356 // this has to be resolved manually.
357 //
358 addConflict(target, source);
359 } else if (!target.hasEqualSemanticAttributes(source)) {
360 // target is modified and is not semantically equal with source. Can't automatically
361 // resolve the differences
362 // => create a conflict
363 addConflict(target, source);
364 } else {
365 // clone from other. mergeFrom will mainly copy
366 // technical attributes like timestamp or user information. Semantic
367 // attributes should already be equal if we get here.
368 //
369 mergeFromSource = true;
370 }
371 if (mergeFromSource) {
372 boolean backupReferrersDownloadedStatus = target.isReferrersDownloaded() && haveSameVersion;
373 target.mergeFrom(source);
374 if (backupReferrersDownloadedStatus && !target.isReferrersDownloaded()) {
375 target.setReferrersDownloaded(true);
376 }
377 objectsWithChildrenToMerge.add(source.getPrimitiveId());
378 }
379 return true;
380 }
381
382 /**
383 * Runs the merge operation. Successfully merged {@link OsmPrimitive}s are in
384 * {@link #getTargetDataSet()}.
385 *
386 * See {@link #getConflicts()} for a map of conflicts after the merge operation.
387 */
388 public void merge() {
389 merge(null);
390 }
391
392 /**
393 * Runs the merge operation. Successfully merged {@link OsmPrimitive}s are in
394 * {@link #getTargetDataSet()}.
395 *
396 * See {@link #getConflicts()} for a map of conflicts after the merge operation.
397 * @param progressMonitor The progress monitor
398 */
399 public void merge(ProgressMonitor progressMonitor) {
400 merge(progressMonitor, true);
401 }
402
403 /**
404 * Runs the merge operation. Successfully merged {@link OsmPrimitive}s are in
405 * {@link #getTargetDataSet()}.
406 *
407 * See {@link #getConflicts()} for a map of conflicts after the merge operation.
408 * @param progressMonitor The progress monitor
409 * @param mergeBounds Whether or not to merge the bounds of the new DataSet to
410 * the existing DataSet
411 * @since 15127
412 */
413 public void merge(ProgressMonitor progressMonitor, boolean mergeBounds) {
414 if (sourceDataSet == null)
415 return;
416 if (progressMonitor != null) {
417 progressMonitor.beginTask(tr("Merging data..."), sourceDataSet.allPrimitives().size());
418 }
419 targetDataSet.update(() -> {
420 List<? extends OsmPrimitive> candidates = null;
421 for (Node node: sourceDataSet.getNodes()) {
422 // lazy initialisation to improve performance, see #19898
423 if (candidates == null) {
424 candidates = new ArrayList<>(targetDataSet.getNodes());
425 }
426 mergePrimitive(node, candidates);
427 if (progressMonitor != null) {
428 progressMonitor.worked(1);
429 }
430 }
431 candidates = null;
432 for (Way way: sourceDataSet.getWays()) {
433 // lazy initialisation to improve performance
434 if (candidates == null) {
435 candidates = new ArrayList<>(targetDataSet.getWays());
436 }
437 mergePrimitive(way, candidates);
438 if (progressMonitor != null) {
439 progressMonitor.worked(1);
440 }
441 }
442 candidates = null;
443 for (Relation relation: sourceDataSet.getRelations()) {
444 // lazy initialisation to improve performance
445 if (candidates == null) {
446 candidates = new ArrayList<>(targetDataSet.getRelations());
447 }
448 mergePrimitive(relation, candidates);
449 if (progressMonitor != null) {
450 progressMonitor.worked(1);
451 }
452 }
453 candidates = null;
454 fixReferences();
455
456 Area a = targetDataSet.getDataSourceArea();
457
458 // copy the merged layer's data source info.
459 // only add source rectangles if they are not contained in the layer already.
460 if (mergeBounds) {
461 for (DataSource src : sourceDataSet.getDataSources()) {
462 if (a == null || !a.contains(src.bounds.asRect())) {
463 targetDataSet.addDataSource(src);
464 }
465 }
466 }
467
468 // copy the merged layer's API version
469 if (targetDataSet.getVersion() == null) {
470 targetDataSet.setVersion(sourceDataSet.getVersion());
471 }
472
473 // copy the merged layer's policies and locked status
474 if (sourceDataSet.getUploadPolicy() != null && (targetDataSet.getUploadPolicy() == null
475 || sourceDataSet.getUploadPolicy().compareTo(targetDataSet.getUploadPolicy()) > 0)) {
476 targetDataSet.setUploadPolicy(sourceDataSet.getUploadPolicy());
477 }
478 if (sourceDataSet.getDownloadPolicy() != null && (targetDataSet.getDownloadPolicy() == null
479 || sourceDataSet.getDownloadPolicy().compareTo(targetDataSet.getDownloadPolicy()) > 0)) {
480 targetDataSet.setDownloadPolicy(sourceDataSet.getDownloadPolicy());
481 }
482 if (sourceDataSet.isLocked() && !targetDataSet.isLocked()) {
483 targetDataSet.lock();
484 }
485 });
486 if (progressMonitor != null) {
487 progressMonitor.finishTask();
488 }
489 }
490
491 /**
492 * replies my dataset
493 *
494 * @return the own (target) data set
495 */
496 public DataSet getTargetDataSet() {
497 return targetDataSet;
498 }
499
500 /**
501 * replies the map of conflicts
502 *
503 * @return the map of conflicts
504 */
505 public ConflictCollection getConflicts() {
506 return conflicts;
507 }
508}
Note: See TracBrowser for help on using the repository browser.