source: josm/trunk/test/functional/org/openstreetmap/josm/io/OsmServerBackreferenceReaderTest.java@ 12850

Last change on this file since 12850 was 12850, checked in by bastiK, 7 years ago

see #15229 - use Config in functional tests
  • Property svn:eol-style set to native
File size: 22.6 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.junit.Assert.assertEquals;
5import static org.junit.Assert.assertFalse;
6import static org.junit.Assert.assertNotNull;
7import static org.junit.Assert.assertTrue;
8import static org.junit.Assert.fail;
9
10import java.io.File;
11import java.io.FileInputStream;
12import java.io.FileNotFoundException;
13import java.io.FileOutputStream;
14import java.io.IOException;
15import java.io.OutputStreamWriter;
16import java.io.PrintWriter;
17import java.nio.charset.StandardCharsets;
18import java.text.MessageFormat;
19import java.util.HashSet;
20import java.util.Locale;
21import java.util.Set;
22import java.util.logging.Logger;
23
24import org.junit.Before;
25import org.junit.BeforeClass;
26import org.junit.Test;
27import org.openstreetmap.josm.JOSMFixture;
28import org.openstreetmap.josm.Main;
29import org.openstreetmap.josm.data.APIDataSet;
30import org.openstreetmap.josm.data.coor.LatLon;
31import org.openstreetmap.josm.data.osm.Changeset;
32import org.openstreetmap.josm.data.osm.CyclicUploadDependencyException;
33import org.openstreetmap.josm.data.osm.DataSet;
34import org.openstreetmap.josm.data.osm.Node;
35import org.openstreetmap.josm.data.osm.OsmPrimitive;
36import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
37import org.openstreetmap.josm.data.osm.Relation;
38import org.openstreetmap.josm.data.osm.RelationMember;
39import org.openstreetmap.josm.data.osm.Way;
40import org.openstreetmap.josm.data.projection.Projections;
41import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
42import org.openstreetmap.josm.spi.preferences.Config;
43import org.openstreetmap.josm.tools.Logging;
44
45import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
46
47/**
48 * Reads primitives referring to a particular primitive (ways including a node, relations referring to a relation)
49 * @since 1806
50 */
51@SuppressFBWarnings(value = "CRLF_INJECTION_LOGS")
52public class OsmServerBackreferenceReaderTest {
53 private static final Logger logger = Logger.getLogger(OsmServerBackreferenceReader.class.getName());
54
55 protected static Node lookupNode(DataSet ds, int i) {
56 for (Node n : ds.getNodes()) {
57 if (("node-" + i).equals(n.get("name"))) return n;
58 }
59 fail("Cannot find node "+i);
60 return null;
61 }
62
63 protected static Way lookupWay(DataSet ds, int i) {
64 for (Way w : ds.getWays()) {
65 if (("way-" + i).equals(w.get("name"))) return w;
66 }
67 fail("Cannot find way "+i);
68 return null;
69 }
70
71 protected static Relation lookupRelation(DataSet ds, int i) {
72 for (Relation r : ds.getRelations()) {
73 if (("relation-" + i).equals(r.get("name"))) return r;
74 }
75 fail("Cannot find relation "+i);
76 return null;
77 }
78
79 protected static void populateTestDataSetWithNodes(DataSet ds) {
80 for (int i = 0; i < 100; i++) {
81 Node n = new Node();
82 n.setCoor(new LatLon(-36.6, 47.6));
83 n.put("name", "node-"+i);
84 ds.addPrimitive(n);
85 }
86 }
87
88 protected static void populateTestDataSetWithWays(DataSet ds) {
89 for (int i = 0; i < 20; i++) {
90 Way w = new Way();
91 for (int j = 0; j < 10; j++) {
92 w.addNode(lookupNode(ds, i+j));
93 }
94 w.put("name", "way-"+i);
95 ds.addPrimitive(w);
96 }
97 }
98
99 protected static void populateTestDataSetWithRelations(DataSet ds) {
100 for (int i = 0; i < 10; i++) {
101 Relation r = new Relation();
102 r.put("name", "relation-" +i);
103 for (int j = 0; j < 10; j++) {
104 RelationMember member = new RelationMember("node-" + j, lookupNode(ds, i + j));
105 r.addMember(member);
106 }
107 for (int j = 0; j < 5; j++) {
108 RelationMember member = new RelationMember("way-" + j, lookupWay(ds, i + j));
109 r.addMember(member);
110 }
111 if (i > 5) {
112 for (int j = 0; j < 3; j++) {
113 RelationMember member = new RelationMember("relation-" + j, lookupRelation(ds, j));
114 logger.info(MessageFormat.format("adding relation {0} to relation {1}", j, i));
115 r.addMember(member);
116 }
117 }
118 ds.addPrimitive(r);
119 }
120 }
121
122 protected static DataSet buildTestDataSet() {
123 DataSet ds = new DataSet();
124 ds.setVersion("0.6");
125
126 populateTestDataSetWithNodes(ds);
127 populateTestDataSetWithWays(ds);
128 populateTestDataSetWithRelations(ds);
129 return ds;
130 }
131
132 /**
133 * creates the dataset on the server.
134 *
135 * @param ds the data set
136 * @throws OsmTransferException if something goes wrong
137 * @throws CyclicUploadDependencyException if a cyclic dependency is detected
138 */
139 public static void createDataSetOnServer(APIDataSet ds) throws OsmTransferException, CyclicUploadDependencyException {
140 logger.info("creating data set on the server ...");
141 ds.adjustRelationUploadOrder();
142 OsmServerWriter writer = new OsmServerWriter();
143 Changeset cs = new Changeset();
144 writer.uploadOsm(
145 new UploadStrategySpecification().setStrategy(UploadStrategy.SINGLE_REQUEST_STRATEGY),
146 ds.getPrimitives(), cs, NullProgressMonitor.INSTANCE);
147 OsmApi.getOsmApi().closeChangeset(cs, NullProgressMonitor.INSTANCE);
148 }
149
150 static DataSet testDataSet;
151
152 /**
153 * Setup test.
154 * @throws OsmTransferException if something goes wrong
155 * @throws CyclicUploadDependencyException if a cyclic dependency is detected
156 * @throws IOException if an I/O error occurs
157 */
158 @BeforeClass
159 public static void setUpBeforeClass() throws OsmTransferException, CyclicUploadDependencyException, IOException {
160 logger.info("initializing ...");
161
162 JOSMFixture.createFunctionalTestFixture().init();
163
164 Config.getPref().put("osm-server.auth-method", "basic");
165
166 // don't use atomic upload, the test API server can't cope with large diff uploads
167 //
168 Config.getPref().putBoolean("osm-server.atomic-upload", false);
169 Main.setProjection(Projections.getProjectionByCode("EPSG:3857")); // Mercator
170 Logging.setLogLevel(Logging.LEVEL_DEBUG);
171
172 File dataSetCacheOutputFile = new File(System.getProperty("java.io.tmpdir"),
173 MultiFetchServerObjectReaderTest.class.getName() + ".dataset");
174
175 String p = System.getProperty("useCachedDataset");
176 if (p != null && Boolean.parseBoolean(p.trim().toLowerCase(Locale.ENGLISH))) {
177 logger.info(MessageFormat.format("property ''{0}'' set, using cached dataset", "useCachedDataset"));
178 return;
179 }
180
181 logger.info(MessageFormat.format(
182 "property ''{0}'' not set to true, creating test dataset on the server. property is ''{1}''", "useCachedDataset", p));
183
184 // build and upload the test data set
185 //
186 logger.info("creating test data set ....");
187 testDataSet = buildTestDataSet();
188 logger.info("uploading test data set ...");
189 createDataSetOnServer(new APIDataSet(testDataSet));
190
191 try (
192 PrintWriter pw = new PrintWriter(
193 new OutputStreamWriter(new FileOutputStream(dataSetCacheOutputFile), StandardCharsets.UTF_8)
194 )) {
195 logger.info(MessageFormat.format("caching test data set in ''{0}'' ...", dataSetCacheOutputFile.toString()));
196 try (OsmWriter w = new OsmWriter(pw, false, testDataSet.getVersion())) {
197 w.header();
198 w.writeDataSources(testDataSet);
199 w.writeContent(testDataSet);
200 w.footer();
201 }
202 }
203 }
204
205 private DataSet ds;
206
207 /**
208 * Setup test.
209 * @throws IOException if any I/O error occurs
210 * @throws IllegalDataException if an error was found while parsing the OSM data
211 * @throws FileNotFoundException if the dataset file cannot be found
212 */
213 @Before
214 public void setUp() throws IOException, IllegalDataException, FileNotFoundException {
215 File f = new File(System.getProperty("java.io.tmpdir"), MultiFetchServerObjectReaderTest.class.getName() + ".dataset");
216 logger.info(MessageFormat.format("reading cached dataset ''{0}''", f.toString()));
217 ds = new DataSet();
218 try (FileInputStream fis = new FileInputStream(f)) {
219 ds = OsmReader.parseDataSet(fis, NullProgressMonitor.INSTANCE);
220 }
221 }
222
223 /**
224 * Test reading references for a node.
225 * @throws OsmTransferException if an error occurs
226 */
227 @Test
228 public void testBackreferenceForNode() throws OsmTransferException {
229 Node n = lookupNode(ds, 0);
230 assertNotNull(n);
231 Way w = lookupWay(ds, 0);
232 assertNotNull(w);
233
234 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(n);
235 reader.setReadFull(false);
236 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
237 printNumberOfPrimitives(referers);
238
239 Set<Long> expectedNodeIds = new HashSet<>();
240 Set<Long> expectedWayIds = new HashSet<>();
241 Set<Long> expectedRelationIds = new HashSet<>();
242
243 for (OsmPrimitive ref : n.getReferrers()) {
244 if (ref instanceof Way) {
245 expectedWayIds.add(ref.getId());
246 expectedNodeIds.addAll(getNodeIdsInWay((Way) ref));
247 } else if (ref instanceof Relation) {
248 expectedRelationIds.add(ref.getId());
249 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, false));
250 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
251 }
252 }
253
254 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
255 assertEquals(expectedWayIds.size(), referers.getWays().size());
256 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
257
258 for (Node node : referers.getNodes()) {
259 assertTrue(expectedNodeIds.contains(node.getId()));
260 assertFalse(node.isIncomplete());
261 }
262
263 for (Way way : referers.getWays()) {
264 assertTrue(expectedWayIds.contains(way.getId()));
265 assertEquals(n.getReferrers().contains(way), !way.isIncomplete());
266 }
267
268 for (Relation relation : referers.getRelations()) {
269 assertTrue(expectedRelationIds.contains(relation.getId()));
270 assertFalse(relation.isIncomplete());
271 }
272 }
273
274 private void printNumberOfPrimitives(DataSet referers) {
275 System.out.println("#nodes=" + referers.getNodes().size() +
276 " #ways=" + referers.getWays().size() +
277 " #relations=" + referers.getRelations().size());
278 }
279
280 /**
281 * Test reading full references for a node.
282 * @throws OsmTransferException if an error occurs
283 */
284 @Test
285 public void testBackreferenceForNodeFull() throws OsmTransferException {
286 Node n = lookupNode(ds, 0);
287 assertNotNull(n);
288
289 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(n);
290 reader.setReadFull(true);
291 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
292 printNumberOfPrimitives(referers);
293
294 Set<Long> expectedNodeIds = new HashSet<>();
295 Set<Long> expectedWayIds = new HashSet<>();
296 Set<Long> expectedRelationIds = new HashSet<>();
297 for (OsmPrimitive ref : n.getReferrers()) {
298 if (ref instanceof Way) {
299 expectedWayIds.add(ref.getId());
300 expectedNodeIds.addAll(getNodeIdsInWay((Way) ref));
301 } else if (ref instanceof Relation) {
302 expectedRelationIds.add(ref.getId());
303 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, true));
304 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, true));
305 }
306 }
307
308 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
309 assertEquals(expectedWayIds.size(), referers.getWays().size());
310 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
311
312 for (Node node : referers.getNodes()) {
313 assertTrue(expectedNodeIds.contains(node.getId()));
314 assertFalse(node.isIncomplete());
315 }
316
317 for (Way way : referers.getWays()) {
318 assertTrue(expectedWayIds.contains(way.getId()));
319 assertFalse(way.isIncomplete());
320 }
321
322 for (Relation relation : referers.getRelations()) {
323 assertTrue(expectedRelationIds.contains(relation.getId()));
324 assertFalse(relation.isIncomplete());
325 }
326 }
327
328 /**
329 * Test reading references for a way.
330 * @throws OsmTransferException if an error occurs
331 */
332 @Test
333 public void testBackreferenceForWay() throws OsmTransferException {
334 Way w = lookupWay(ds, 1);
335 assertNotNull(w);
336 // way with name "way-1" is referred to by two relations
337 //
338
339 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(w);
340 reader.setReadFull(false);
341 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
342 printNumberOfPrimitives(referers);
343
344 Set<Long> expectedNodeIds = new HashSet<>();
345 Set<Long> expectedWayIds = new HashSet<>();
346 Set<Long> expectedRelationIds = new HashSet<>();
347
348 for (OsmPrimitive ref : w.getReferrers()) {
349 if (ref instanceof Relation) {
350 expectedRelationIds.add(ref.getId());
351 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, false));
352 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
353 }
354 }
355
356 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
357 assertEquals(expectedWayIds.size(), referers.getWays().size());
358 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
359
360 for (Way w1 : referers.getWays()) {
361 assertTrue(w1.isIncomplete());
362 }
363 assertEquals(2, referers.getRelations().size()); // two relations referring to w
364
365 Relation r = lookupRelation(referers, 0);
366 assertNotNull(r);
367 assertFalse(r.isIncomplete());
368 r = lookupRelation(referers, 1);
369 assertFalse(r.isIncomplete());
370 }
371
372 /**
373 * Test reading full references for a way.
374 * @throws OsmTransferException if an error occurs
375 */
376 @Test
377 public void testBackreferenceForWayFull() throws OsmTransferException {
378 Way w = lookupWay(ds, 1);
379 assertNotNull(w);
380 // way with name "way-1" is referred to by two relations
381 //
382
383 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(w);
384 reader.setReadFull(true);
385 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
386 assertEquals(6, referers.getWays().size()); // 6 ways referred by two relations
387 for (Way w1 : referers.getWays()) {
388 assertFalse(w1.isIncomplete());
389 }
390 assertEquals(2, referers.getRelations().size()); // two relations referring to
391 Set<Long> expectedNodeIds = new HashSet<>();
392 for (Way way : referers.getWays()) {
393 Way orig = (Way) ds.getPrimitiveById(way);
394 for (Node n : orig.getNodes()) {
395 expectedNodeIds.add(n.getId());
396 }
397 }
398 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
399 for (Node n : referers.getNodes()) {
400 assertTrue(expectedNodeIds.contains(n.getId()));
401 }
402
403 Relation r = lookupRelation(referers, 0);
404 assertNotNull(r);
405 assertFalse(r.isIncomplete());
406 r = lookupRelation(referers, 1);
407 assertFalse(r.isIncomplete());
408 }
409
410 /**
411 * Test reading references for a relation.
412 * @throws OsmTransferException if an error occurs
413 */
414 @Test
415 public void testBackreferenceForRelation() throws OsmTransferException {
416 Relation r = lookupRelation(ds, 1);
417 assertNotNull(r);
418 // way with name "relation-1" is referred to by four relations:
419 // relation-6, relation-7, relation-8, relation-9
420 //
421
422 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(r);
423 reader.setReadFull(false);
424 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
425 printNumberOfPrimitives(referers);
426
427 Set<Long> referringRelationsIds = new HashSet<>();
428 Relation r6 = lookupRelation(referers, 6);
429 assertNotNull(r6);
430 assertFalse(r6.isIncomplete());
431 referringRelationsIds.add(r6.getId());
432 Relation r7 = lookupRelation(referers, 7);
433 assertNotNull(r7);
434 assertFalse(r7.isIncomplete());
435 referringRelationsIds.add(r7.getId());
436 Relation r8 = lookupRelation(referers, 8);
437 assertNotNull(r8);
438 assertFalse(r8.isIncomplete());
439 referringRelationsIds.add(r8.getId());
440 Relation r9 = lookupRelation(referers, 9);
441 assertNotNull(r9);
442 assertFalse(r9.isIncomplete());
443 referringRelationsIds.add(r9.getId());
444
445 for (Relation r1 : referers.getRelations()) {
446 if (!referringRelationsIds.contains(r1.getId())) {
447 assertTrue(r1.isIncomplete());
448 }
449 }
450
451 // make sure we read all ways referred to by parent relations. These
452 // ways are incomplete after reading.
453 //
454 Set<Long> expectedWayIds = new HashSet<>();
455 for (RelationMember m : lookupRelation(ds, 6).getMembers()) {
456 if (m.isWay()) {
457 expectedWayIds.add(m.getMember().getId());
458 }
459 }
460 for (RelationMember m : lookupRelation(ds, 7).getMembers()) {
461 if (m.isWay()) {
462 expectedWayIds.add(m.getMember().getId());
463 }
464 }
465 for (RelationMember m : lookupRelation(ds, 8).getMembers()) {
466 if (m.isWay()) {
467 expectedWayIds.add(m.getMember().getId());
468 }
469 }
470 for (RelationMember m : lookupRelation(ds, 9).getMembers()) {
471 if (m.isWay()) {
472 expectedWayIds.add(m.getMember().getId());
473 }
474 }
475
476 assertEquals(expectedWayIds.size(), referers.getWays().size());
477 for (Way w1 : referers.getWays()) {
478 assertTrue(expectedWayIds.contains(w1.getId()));
479 assertTrue(w1.isIncomplete());
480 }
481
482 // make sure we read all nodes referred to by parent relations.
483 Set<Long> expectedNodeIds = new HashSet<>();
484 for (OsmPrimitive ref : r.getReferrers()) {
485 if (ref instanceof Relation) {
486 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
487 }
488 }
489 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
490 }
491
492 protected static Set<Long> getNodeIdsInWay(Way way) {
493 HashSet<Long> ret = new HashSet<>();
494 if (way == null) return ret;
495 for (Node n: way.getNodes()) {
496 ret.add(n.getId());
497 }
498 return ret;
499 }
500
501 protected static Set<Long> getNodeIdsInRelation(Relation r, boolean children) {
502 HashSet<Long> ret = new HashSet<>();
503 if (r == null) return ret;
504 for (RelationMember m: r.getMembers()) {
505 if (m.isNode()) {
506 ret.add(m.getMember().getId());
507 } else if (m.isWay() && children) {
508 ret.addAll(getNodeIdsInWay(m.getWay()));
509 } else if (m.isRelation() && children) {
510 ret.addAll(getNodeIdsInRelation(m.getRelation(), true));
511 }
512 }
513 return ret;
514 }
515
516 protected static Set<Long> getWayIdsInRelation(Relation r, boolean children) {
517 HashSet<Long> ret = new HashSet<>();
518 if (r == null) return ret;
519 for (RelationMember m: r.getMembers()) {
520 if (m.isWay()) {
521 ret.add(m.getMember().getId());
522 } else if (m.isRelation() && children) {
523 ret.addAll(getWayIdsInRelation(m.getRelation(), true));
524 }
525 }
526 return ret;
527 }
528
529 /**
530 * Test reading full references for a relation.
531 * @throws OsmTransferException if an error occurs
532 */
533 @Test
534 public void testBackreferenceForRelationFull() throws OsmTransferException {
535 Relation r = lookupRelation(ds, 1);
536 assertNotNull(r);
537 // way with name "relation-1" is referred to by four relations:
538 // relation-6, relation-7, relation-8, relation-9
539 //
540
541 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(r);
542 reader.setReadFull(true);
543 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
544
545 r = lookupRelation(referers, 6);
546 assertNotNull(r);
547 assertFalse(r.isIncomplete());
548 r = lookupRelation(referers, 7);
549 assertNotNull(r);
550 assertFalse(r.isIncomplete());
551 r = lookupRelation(referers, 8);
552 assertNotNull(r);
553 assertFalse(r.isIncomplete());
554 r = lookupRelation(referers, 9);
555 assertNotNull(r);
556 assertFalse(r.isIncomplete());
557
558 // all relations are fully loaded
559 //
560 for (Relation r1 : referers.getRelations()) {
561 assertFalse(r1.isIncomplete());
562 }
563
564 // make sure we read all ways referred to by parent relations. These
565 // ways are completely read after reading the relations
566 //
567 Set<Long> expectedWayIds = new HashSet<>();
568 for (RelationMember m : lookupRelation(ds, 6).getMembers()) {
569 if (m.isWay()) {
570 expectedWayIds.add(m.getMember().getId());
571 }
572 }
573 for (RelationMember m : lookupRelation(ds, 7).getMembers()) {
574 if (m.isWay()) {
575 expectedWayIds.add(m.getMember().getId());
576 }
577 }
578 for (RelationMember m : lookupRelation(ds, 8).getMembers()) {
579 if (m.isWay()) {
580 expectedWayIds.add(m.getMember().getId());
581 }
582 }
583 for (RelationMember m : lookupRelation(ds, 9).getMembers()) {
584 if (m.isWay()) {
585 expectedWayIds.add(m.getMember().getId());
586 }
587 }
588 for (long id : expectedWayIds) {
589 Way w = (Way) referers.getPrimitiveById(id, OsmPrimitiveType.WAY);
590 assertNotNull(w);
591 assertFalse(w.isIncomplete());
592 }
593
594 Set<Long> expectedNodeIds = new HashSet<>();
595 for (int i = 6; i < 10; i++) {
596 Relation r1 = lookupRelation(ds, i);
597 expectedNodeIds.addAll(getNodeIdsInRelation(r1, true));
598 }
599
600 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
601 for (Node n : referers.getNodes()) {
602 assertTrue(expectedNodeIds.contains(n.getId()));
603 }
604 }
605}
Note: See TracBrowser for help on using the repository browser.