source: josm/trunk/test/functional/org/openstreetmap/josm/io/OsmServerBackreferenceReaderTest.java@ 12673

Last change on this file since 12673 was 12673, checked in by Don-vip, 7 years ago

see #15182 - move CyclicUploadDependencyException from actions.upload to data.osm (used in data.APIDataSet)
  • Property svn:eol-style set to native
File size: 22.6 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.junit.Assert.assertEquals;
5import static org.junit.Assert.assertFalse;
6import static org.junit.Assert.assertNotNull;
7import static org.junit.Assert.assertTrue;
8import static org.junit.Assert.fail;
9
10import java.io.File;
11import java.io.FileInputStream;
12import java.io.FileNotFoundException;
13import java.io.FileOutputStream;
14import java.io.IOException;
15import java.io.OutputStreamWriter;
16import java.io.PrintWriter;
17import java.nio.charset.StandardCharsets;
18import java.text.MessageFormat;
19import java.util.HashSet;
20import java.util.Locale;
21import java.util.Set;
22import java.util.logging.Logger;
23
24import org.junit.Before;
25import org.junit.BeforeClass;
26import org.junit.Test;
27import org.openstreetmap.josm.JOSMFixture;
28import org.openstreetmap.josm.Main;
29import org.openstreetmap.josm.data.APIDataSet;
30import org.openstreetmap.josm.data.coor.LatLon;
31import org.openstreetmap.josm.data.osm.Changeset;
32import org.openstreetmap.josm.data.osm.CyclicUploadDependencyException;
33import org.openstreetmap.josm.data.osm.DataSet;
34import org.openstreetmap.josm.data.osm.Node;
35import org.openstreetmap.josm.data.osm.OsmPrimitive;
36import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
37import org.openstreetmap.josm.data.osm.Relation;
38import org.openstreetmap.josm.data.osm.RelationMember;
39import org.openstreetmap.josm.data.osm.Way;
40import org.openstreetmap.josm.data.projection.Projections;
41import org.openstreetmap.josm.gui.io.UploadStrategy;
42import org.openstreetmap.josm.gui.io.UploadStrategySpecification;
43import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
44import org.openstreetmap.josm.tools.Logging;
45
46import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
47
48/**
49 * Reads primitives referring to a particular primitive (ways including a node, relations referring to a relation)
50 * @since 1806
51 */
52@SuppressFBWarnings(value = "CRLF_INJECTION_LOGS")
53public class OsmServerBackreferenceReaderTest {
54 private static final Logger logger = Logger.getLogger(OsmServerBackreferenceReader.class.getName());
55
56 protected static Node lookupNode(DataSet ds, int i) {
57 for (Node n : ds.getNodes()) {
58 if (("node-" + i).equals(n.get("name"))) return n;
59 }
60 fail("Cannot find node "+i);
61 return null;
62 }
63
64 protected static Way lookupWay(DataSet ds, int i) {
65 for (Way w : ds.getWays()) {
66 if (("way-" + i).equals(w.get("name"))) return w;
67 }
68 fail("Cannot find way "+i);
69 return null;
70 }
71
72 protected static Relation lookupRelation(DataSet ds, int i) {
73 for (Relation r : ds.getRelations()) {
74 if (("relation-" + i).equals(r.get("name"))) return r;
75 }
76 fail("Cannot find relation "+i);
77 return null;
78 }
79
80 protected static void populateTestDataSetWithNodes(DataSet ds) {
81 for (int i = 0; i < 100; i++) {
82 Node n = new Node();
83 n.setCoor(new LatLon(-36.6, 47.6));
84 n.put("name", "node-"+i);
85 ds.addPrimitive(n);
86 }
87 }
88
89 protected static void populateTestDataSetWithWays(DataSet ds) {
90 for (int i = 0; i < 20; i++) {
91 Way w = new Way();
92 for (int j = 0; j < 10; j++) {
93 w.addNode(lookupNode(ds, i+j));
94 }
95 w.put("name", "way-"+i);
96 ds.addPrimitive(w);
97 }
98 }
99
100 protected static void populateTestDataSetWithRelations(DataSet ds) {
101 for (int i = 0; i < 10; i++) {
102 Relation r = new Relation();
103 r.put("name", "relation-" +i);
104 for (int j = 0; j < 10; j++) {
105 RelationMember member = new RelationMember("node-" + j, lookupNode(ds, i + j));
106 r.addMember(member);
107 }
108 for (int j = 0; j < 5; j++) {
109 RelationMember member = new RelationMember("way-" + j, lookupWay(ds, i + j));
110 r.addMember(member);
111 }
112 if (i > 5) {
113 for (int j = 0; j < 3; j++) {
114 RelationMember member = new RelationMember("relation-" + j, lookupRelation(ds, j));
115 logger.info(MessageFormat.format("adding relation {0} to relation {1}", j, i));
116 r.addMember(member);
117 }
118 }
119 ds.addPrimitive(r);
120 }
121 }
122
123 protected static DataSet buildTestDataSet() {
124 DataSet ds = new DataSet();
125 ds.setVersion("0.6");
126
127 populateTestDataSetWithNodes(ds);
128 populateTestDataSetWithWays(ds);
129 populateTestDataSetWithRelations(ds);
130 return ds;
131 }
132
133 /**
134 * creates the dataset on the server.
135 *
136 * @param ds the data set
137 * @throws OsmTransferException if something goes wrong
138 * @throws CyclicUploadDependencyException if a cyclic dependency is detected
139 */
140 public static void createDataSetOnServer(APIDataSet ds) throws OsmTransferException, CyclicUploadDependencyException {
141 logger.info("creating data set on the server ...");
142 ds.adjustRelationUploadOrder();
143 OsmServerWriter writer = new OsmServerWriter();
144 Changeset cs = new Changeset();
145 writer.uploadOsm(
146 new UploadStrategySpecification().setStrategy(UploadStrategy.SINGLE_REQUEST_STRATEGY),
147 ds.getPrimitives(), cs, NullProgressMonitor.INSTANCE);
148 OsmApi.getOsmApi().closeChangeset(cs, NullProgressMonitor.INSTANCE);
149 }
150
151 static DataSet testDataSet;
152
153 /**
154 * Setup test.
155 * @throws OsmTransferException if something goes wrong
156 * @throws CyclicUploadDependencyException if a cyclic dependency is detected
157 * @throws IOException if an I/O error occurs
158 */
159 @BeforeClass
160 public static void setUpBeforeClass() throws OsmTransferException, CyclicUploadDependencyException, IOException {
161 logger.info("initializing ...");
162
163 JOSMFixture.createFunctionalTestFixture().init();
164
165 Main.pref.put("osm-server.auth-method", "basic");
166
167 // don't use atomic upload, the test API server can't cope with large diff uploads
168 //
169 Main.pref.put("osm-server.atomic-upload", false);
170 Main.setProjection(Projections.getProjectionByCode("EPSG:3857")); // Mercator
171 Logging.setLogLevel(Logging.LEVEL_DEBUG);
172
173 File dataSetCacheOutputFile = new File(System.getProperty("java.io.tmpdir"),
174 MultiFetchServerObjectReaderTest.class.getName() + ".dataset");
175
176 String p = System.getProperty("useCachedDataset");
177 if (p != null && Boolean.parseBoolean(p.trim().toLowerCase(Locale.ENGLISH))) {
178 logger.info(MessageFormat.format("property ''{0}'' set, using cached dataset", "useCachedDataset"));
179 return;
180 }
181
182 logger.info(MessageFormat.format(
183 "property ''{0}'' not set to true, creating test dataset on the server. property is ''{1}''", "useCachedDataset", p));
184
185 // build and upload the test data set
186 //
187 logger.info("creating test data set ....");
188 testDataSet = buildTestDataSet();
189 logger.info("uploading test data set ...");
190 createDataSetOnServer(new APIDataSet(testDataSet));
191
192 try (
193 PrintWriter pw = new PrintWriter(
194 new OutputStreamWriter(new FileOutputStream(dataSetCacheOutputFile), StandardCharsets.UTF_8)
195 )) {
196 logger.info(MessageFormat.format("caching test data set in ''{0}'' ...", dataSetCacheOutputFile.toString()));
197 try (OsmWriter w = new OsmWriter(pw, false, testDataSet.getVersion())) {
198 w.header();
199 w.writeDataSources(testDataSet);
200 w.writeContent(testDataSet);
201 w.footer();
202 }
203 }
204 }
205
206 private DataSet ds;
207
208 /**
209 * Setup test.
210 * @throws IOException if any I/O error occurs
211 * @throws IllegalDataException if an error was found while parsing the OSM data
212 * @throws FileNotFoundException if the dataset file cannot be found
213 */
214 @Before
215 public void setUp() throws IOException, IllegalDataException, FileNotFoundException {
216 File f = new File(System.getProperty("java.io.tmpdir"), MultiFetchServerObjectReaderTest.class.getName() + ".dataset");
217 logger.info(MessageFormat.format("reading cached dataset ''{0}''", f.toString()));
218 ds = new DataSet();
219 try (FileInputStream fis = new FileInputStream(f)) {
220 ds = OsmReader.parseDataSet(fis, NullProgressMonitor.INSTANCE);
221 }
222 }
223
224 /**
225 * Test reading references for a node.
226 * @throws OsmTransferException if an error occurs
227 */
228 @Test
229 public void testBackreferenceForNode() throws OsmTransferException {
230 Node n = lookupNode(ds, 0);
231 assertNotNull(n);
232 Way w = lookupWay(ds, 0);
233 assertNotNull(w);
234
235 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(n);
236 reader.setReadFull(false);
237 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
238 printNumberOfPrimitives(referers);
239
240 Set<Long> expectedNodeIds = new HashSet<>();
241 Set<Long> expectedWayIds = new HashSet<>();
242 Set<Long> expectedRelationIds = new HashSet<>();
243
244 for (OsmPrimitive ref : n.getReferrers()) {
245 if (ref instanceof Way) {
246 expectedWayIds.add(ref.getId());
247 expectedNodeIds.addAll(getNodeIdsInWay((Way) ref));
248 } else if (ref instanceof Relation) {
249 expectedRelationIds.add(ref.getId());
250 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, false));
251 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
252 }
253 }
254
255 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
256 assertEquals(expectedWayIds.size(), referers.getWays().size());
257 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
258
259 for (Node node : referers.getNodes()) {
260 assertTrue(expectedNodeIds.contains(node.getId()));
261 assertFalse(node.isIncomplete());
262 }
263
264 for (Way way : referers.getWays()) {
265 assertTrue(expectedWayIds.contains(way.getId()));
266 assertEquals(n.getReferrers().contains(way), !way.isIncomplete());
267 }
268
269 for (Relation relation : referers.getRelations()) {
270 assertTrue(expectedRelationIds.contains(relation.getId()));
271 assertFalse(relation.isIncomplete());
272 }
273 }
274
275 private void printNumberOfPrimitives(DataSet referers) {
276 System.out.println("#nodes=" + referers.getNodes().size() +
277 " #ways=" + referers.getWays().size() +
278 " #relations=" + referers.getRelations().size());
279 }
280
281 /**
282 * Test reading full references for a node.
283 * @throws OsmTransferException if an error occurs
284 */
285 @Test
286 public void testBackreferenceForNodeFull() throws OsmTransferException {
287 Node n = lookupNode(ds, 0);
288 assertNotNull(n);
289
290 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(n);
291 reader.setReadFull(true);
292 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
293 printNumberOfPrimitives(referers);
294
295 Set<Long> expectedNodeIds = new HashSet<>();
296 Set<Long> expectedWayIds = new HashSet<>();
297 Set<Long> expectedRelationIds = new HashSet<>();
298 for (OsmPrimitive ref : n.getReferrers()) {
299 if (ref instanceof Way) {
300 expectedWayIds.add(ref.getId());
301 expectedNodeIds.addAll(getNodeIdsInWay((Way) ref));
302 } else if (ref instanceof Relation) {
303 expectedRelationIds.add(ref.getId());
304 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, true));
305 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, true));
306 }
307 }
308
309 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
310 assertEquals(expectedWayIds.size(), referers.getWays().size());
311 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
312
313 for (Node node : referers.getNodes()) {
314 assertTrue(expectedNodeIds.contains(node.getId()));
315 assertFalse(node.isIncomplete());
316 }
317
318 for (Way way : referers.getWays()) {
319 assertTrue(expectedWayIds.contains(way.getId()));
320 assertFalse(way.isIncomplete());
321 }
322
323 for (Relation relation : referers.getRelations()) {
324 assertTrue(expectedRelationIds.contains(relation.getId()));
325 assertFalse(relation.isIncomplete());
326 }
327 }
328
329 /**
330 * Test reading references for a way.
331 * @throws OsmTransferException if an error occurs
332 */
333 @Test
334 public void testBackreferenceForWay() throws OsmTransferException {
335 Way w = lookupWay(ds, 1);
336 assertNotNull(w);
337 // way with name "way-1" is referred to by two relations
338 //
339
340 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(w);
341 reader.setReadFull(false);
342 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
343 printNumberOfPrimitives(referers);
344
345 Set<Long> expectedNodeIds = new HashSet<>();
346 Set<Long> expectedWayIds = new HashSet<>();
347 Set<Long> expectedRelationIds = new HashSet<>();
348
349 for (OsmPrimitive ref : w.getReferrers()) {
350 if (ref instanceof Relation) {
351 expectedRelationIds.add(ref.getId());
352 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, false));
353 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
354 }
355 }
356
357 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
358 assertEquals(expectedWayIds.size(), referers.getWays().size());
359 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
360
361 for (Way w1 : referers.getWays()) {
362 assertTrue(w1.isIncomplete());
363 }
364 assertEquals(2, referers.getRelations().size()); // two relations referring to w
365
366 Relation r = lookupRelation(referers, 0);
367 assertNotNull(r);
368 assertFalse(r.isIncomplete());
369 r = lookupRelation(referers, 1);
370 assertFalse(r.isIncomplete());
371 }
372
373 /**
374 * Test reading full references for a way.
375 * @throws OsmTransferException if an error occurs
376 */
377 @Test
378 public void testBackreferenceForWayFull() throws OsmTransferException {
379 Way w = lookupWay(ds, 1);
380 assertNotNull(w);
381 // way with name "way-1" is referred to by two relations
382 //
383
384 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(w);
385 reader.setReadFull(true);
386 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
387 assertEquals(6, referers.getWays().size()); // 6 ways referred by two relations
388 for (Way w1 : referers.getWays()) {
389 assertFalse(w1.isIncomplete());
390 }
391 assertEquals(2, referers.getRelations().size()); // two relations referring to
392 Set<Long> expectedNodeIds = new HashSet<>();
393 for (Way way : referers.getWays()) {
394 Way orig = (Way) ds.getPrimitiveById(way);
395 for (Node n : orig.getNodes()) {
396 expectedNodeIds.add(n.getId());
397 }
398 }
399 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
400 for (Node n : referers.getNodes()) {
401 assertTrue(expectedNodeIds.contains(n.getId()));
402 }
403
404 Relation r = lookupRelation(referers, 0);
405 assertNotNull(r);
406 assertFalse(r.isIncomplete());
407 r = lookupRelation(referers, 1);
408 assertFalse(r.isIncomplete());
409 }
410
411 /**
412 * Test reading references for a relation.
413 * @throws OsmTransferException if an error occurs
414 */
415 @Test
416 public void testBackreferenceForRelation() throws OsmTransferException {
417 Relation r = lookupRelation(ds, 1);
418 assertNotNull(r);
419 // way with name "relation-1" is referred to by four relations:
420 // relation-6, relation-7, relation-8, relation-9
421 //
422
423 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(r);
424 reader.setReadFull(false);
425 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
426 printNumberOfPrimitives(referers);
427
428 Set<Long> referringRelationsIds = new HashSet<>();
429 Relation r6 = lookupRelation(referers, 6);
430 assertNotNull(r6);
431 assertFalse(r6.isIncomplete());
432 referringRelationsIds.add(r6.getId());
433 Relation r7 = lookupRelation(referers, 7);
434 assertNotNull(r7);
435 assertFalse(r7.isIncomplete());
436 referringRelationsIds.add(r7.getId());
437 Relation r8 = lookupRelation(referers, 8);
438 assertNotNull(r8);
439 assertFalse(r8.isIncomplete());
440 referringRelationsIds.add(r8.getId());
441 Relation r9 = lookupRelation(referers, 9);
442 assertNotNull(r9);
443 assertFalse(r9.isIncomplete());
444 referringRelationsIds.add(r9.getId());
445
446 for (Relation r1 : referers.getRelations()) {
447 if (!referringRelationsIds.contains(r1.getId())) {
448 assertTrue(r1.isIncomplete());
449 }
450 }
451
452 // make sure we read all ways referred to by parent relations. These
453 // ways are incomplete after reading.
454 //
455 Set<Long> expectedWayIds = new HashSet<>();
456 for (RelationMember m : lookupRelation(ds, 6).getMembers()) {
457 if (m.isWay()) {
458 expectedWayIds.add(m.getMember().getId());
459 }
460 }
461 for (RelationMember m : lookupRelation(ds, 7).getMembers()) {
462 if (m.isWay()) {
463 expectedWayIds.add(m.getMember().getId());
464 }
465 }
466 for (RelationMember m : lookupRelation(ds, 8).getMembers()) {
467 if (m.isWay()) {
468 expectedWayIds.add(m.getMember().getId());
469 }
470 }
471 for (RelationMember m : lookupRelation(ds, 9).getMembers()) {
472 if (m.isWay()) {
473 expectedWayIds.add(m.getMember().getId());
474 }
475 }
476
477 assertEquals(expectedWayIds.size(), referers.getWays().size());
478 for (Way w1 : referers.getWays()) {
479 assertTrue(expectedWayIds.contains(w1.getId()));
480 assertTrue(w1.isIncomplete());
481 }
482
483 // make sure we read all nodes referred to by parent relations.
484 Set<Long> expectedNodeIds = new HashSet<>();
485 for (OsmPrimitive ref : r.getReferrers()) {
486 if (ref instanceof Relation) {
487 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
488 }
489 }
490 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
491 }
492
493 protected static Set<Long> getNodeIdsInWay(Way way) {
494 HashSet<Long> ret = new HashSet<>();
495 if (way == null) return ret;
496 for (Node n: way.getNodes()) {
497 ret.add(n.getId());
498 }
499 return ret;
500 }
501
502 protected static Set<Long> getNodeIdsInRelation(Relation r, boolean children) {
503 HashSet<Long> ret = new HashSet<>();
504 if (r == null) return ret;
505 for (RelationMember m: r.getMembers()) {
506 if (m.isNode()) {
507 ret.add(m.getMember().getId());
508 } else if (m.isWay() && children) {
509 ret.addAll(getNodeIdsInWay(m.getWay()));
510 } else if (m.isRelation() && children) {
511 ret.addAll(getNodeIdsInRelation(m.getRelation(), true));
512 }
513 }
514 return ret;
515 }
516
517 protected static Set<Long> getWayIdsInRelation(Relation r, boolean children) {
518 HashSet<Long> ret = new HashSet<>();
519 if (r == null) return ret;
520 for (RelationMember m: r.getMembers()) {
521 if (m.isWay()) {
522 ret.add(m.getMember().getId());
523 } else if (m.isRelation() && children) {
524 ret.addAll(getWayIdsInRelation(m.getRelation(), true));
525 }
526 }
527 return ret;
528 }
529
530 /**
531 * Test reading full references for a relation.
532 * @throws OsmTransferException if an error occurs
533 */
534 @Test
535 public void testBackreferenceForRelationFull() throws OsmTransferException {
536 Relation r = lookupRelation(ds, 1);
537 assertNotNull(r);
538 // way with name "relation-1" is referred to by four relations:
539 // relation-6, relation-7, relation-8, relation-9
540 //
541
542 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(r);
543 reader.setReadFull(true);
544 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
545
546 r = lookupRelation(referers, 6);
547 assertNotNull(r);
548 assertFalse(r.isIncomplete());
549 r = lookupRelation(referers, 7);
550 assertNotNull(r);
551 assertFalse(r.isIncomplete());
552 r = lookupRelation(referers, 8);
553 assertNotNull(r);
554 assertFalse(r.isIncomplete());
555 r = lookupRelation(referers, 9);
556 assertNotNull(r);
557 assertFalse(r.isIncomplete());
558
559 // all relations are fully loaded
560 //
561 for (Relation r1 : referers.getRelations()) {
562 assertFalse(r1.isIncomplete());
563 }
564
565 // make sure we read all ways referred to by parent relations. These
566 // ways are completely read after reading the relations
567 //
568 Set<Long> expectedWayIds = new HashSet<>();
569 for (RelationMember m : lookupRelation(ds, 6).getMembers()) {
570 if (m.isWay()) {
571 expectedWayIds.add(m.getMember().getId());
572 }
573 }
574 for (RelationMember m : lookupRelation(ds, 7).getMembers()) {
575 if (m.isWay()) {
576 expectedWayIds.add(m.getMember().getId());
577 }
578 }
579 for (RelationMember m : lookupRelation(ds, 8).getMembers()) {
580 if (m.isWay()) {
581 expectedWayIds.add(m.getMember().getId());
582 }
583 }
584 for (RelationMember m : lookupRelation(ds, 9).getMembers()) {
585 if (m.isWay()) {
586 expectedWayIds.add(m.getMember().getId());
587 }
588 }
589 for (long id : expectedWayIds) {
590 Way w = (Way) referers.getPrimitiveById(id, OsmPrimitiveType.WAY);
591 assertNotNull(w);
592 assertFalse(w.isIncomplete());
593 }
594
595 Set<Long> expectedNodeIds = new HashSet<>();
596 for (int i = 6; i < 10; i++) {
597 Relation r1 = lookupRelation(ds, i);
598 expectedNodeIds.addAll(getNodeIdsInRelation(r1, true));
599 }
600
601 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
602 for (Node n : referers.getNodes()) {
603 assertTrue(expectedNodeIds.contains(n.getId()));
604 }
605 }
606}
Note: See TracBrowser for help on using the repository browser.