source: josm/trunk/test/functional/org/openstreetmap/josm/io/OsmServerBackreferenceReaderTest.java@ 12687

Last change on this file since 12687 was 12687, checked in by Don-vip, 7 years ago

see #15182 - move UploadStrategySpecification and required enums from gui.io to io
  • Property svn:eol-style set to native
File size: 22.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.junit.Assert.assertEquals;
5import static org.junit.Assert.assertFalse;
6import static org.junit.Assert.assertNotNull;
7import static org.junit.Assert.assertTrue;
8import static org.junit.Assert.fail;
9
10import java.io.File;
11import java.io.FileInputStream;
12import java.io.FileNotFoundException;
13import java.io.FileOutputStream;
14import java.io.IOException;
15import java.io.OutputStreamWriter;
16import java.io.PrintWriter;
17import java.nio.charset.StandardCharsets;
18import java.text.MessageFormat;
19import java.util.HashSet;
20import java.util.Locale;
21import java.util.Set;
22import java.util.logging.Logger;
23
24import org.junit.Before;
25import org.junit.BeforeClass;
26import org.junit.Test;
27import org.openstreetmap.josm.JOSMFixture;
28import org.openstreetmap.josm.Main;
29import org.openstreetmap.josm.data.APIDataSet;
30import org.openstreetmap.josm.data.coor.LatLon;
31import org.openstreetmap.josm.data.osm.Changeset;
32import org.openstreetmap.josm.data.osm.CyclicUploadDependencyException;
33import org.openstreetmap.josm.data.osm.DataSet;
34import org.openstreetmap.josm.data.osm.Node;
35import org.openstreetmap.josm.data.osm.OsmPrimitive;
36import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
37import org.openstreetmap.josm.data.osm.Relation;
38import org.openstreetmap.josm.data.osm.RelationMember;
39import org.openstreetmap.josm.data.osm.Way;
40import org.openstreetmap.josm.data.projection.Projections;
41import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
42import org.openstreetmap.josm.tools.Logging;
43
44import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
45
46/**
47 * Reads primitives referring to a particular primitive (ways including a node, relations referring to a relation)
48 * @since 1806
49 */
50@SuppressFBWarnings(value = "CRLF_INJECTION_LOGS")
51public class OsmServerBackreferenceReaderTest {
52 private static final Logger logger = Logger.getLogger(OsmServerBackreferenceReader.class.getName());
53
54 protected static Node lookupNode(DataSet ds, int i) {
55 for (Node n : ds.getNodes()) {
56 if (("node-" + i).equals(n.get("name"))) return n;
57 }
58 fail("Cannot find node "+i);
59 return null;
60 }
61
62 protected static Way lookupWay(DataSet ds, int i) {
63 for (Way w : ds.getWays()) {
64 if (("way-" + i).equals(w.get("name"))) return w;
65 }
66 fail("Cannot find way "+i);
67 return null;
68 }
69
70 protected static Relation lookupRelation(DataSet ds, int i) {
71 for (Relation r : ds.getRelations()) {
72 if (("relation-" + i).equals(r.get("name"))) return r;
73 }
74 fail("Cannot find relation "+i);
75 return null;
76 }
77
78 protected static void populateTestDataSetWithNodes(DataSet ds) {
79 for (int i = 0; i < 100; i++) {
80 Node n = new Node();
81 n.setCoor(new LatLon(-36.6, 47.6));
82 n.put("name", "node-"+i);
83 ds.addPrimitive(n);
84 }
85 }
86
87 protected static void populateTestDataSetWithWays(DataSet ds) {
88 for (int i = 0; i < 20; i++) {
89 Way w = new Way();
90 for (int j = 0; j < 10; j++) {
91 w.addNode(lookupNode(ds, i+j));
92 }
93 w.put("name", "way-"+i);
94 ds.addPrimitive(w);
95 }
96 }
97
98 protected static void populateTestDataSetWithRelations(DataSet ds) {
99 for (int i = 0; i < 10; i++) {
100 Relation r = new Relation();
101 r.put("name", "relation-" +i);
102 for (int j = 0; j < 10; j++) {
103 RelationMember member = new RelationMember("node-" + j, lookupNode(ds, i + j));
104 r.addMember(member);
105 }
106 for (int j = 0; j < 5; j++) {
107 RelationMember member = new RelationMember("way-" + j, lookupWay(ds, i + j));
108 r.addMember(member);
109 }
110 if (i > 5) {
111 for (int j = 0; j < 3; j++) {
112 RelationMember member = new RelationMember("relation-" + j, lookupRelation(ds, j));
113 logger.info(MessageFormat.format("adding relation {0} to relation {1}", j, i));
114 r.addMember(member);
115 }
116 }
117 ds.addPrimitive(r);
118 }
119 }
120
121 protected static DataSet buildTestDataSet() {
122 DataSet ds = new DataSet();
123 ds.setVersion("0.6");
124
125 populateTestDataSetWithNodes(ds);
126 populateTestDataSetWithWays(ds);
127 populateTestDataSetWithRelations(ds);
128 return ds;
129 }
130
131 /**
132 * creates the dataset on the server.
133 *
134 * @param ds the data set
135 * @throws OsmTransferException if something goes wrong
136 * @throws CyclicUploadDependencyException if a cyclic dependency is detected
137 */
138 public static void createDataSetOnServer(APIDataSet ds) throws OsmTransferException, CyclicUploadDependencyException {
139 logger.info("creating data set on the server ...");
140 ds.adjustRelationUploadOrder();
141 OsmServerWriter writer = new OsmServerWriter();
142 Changeset cs = new Changeset();
143 writer.uploadOsm(
144 new UploadStrategySpecification().setStrategy(UploadStrategy.SINGLE_REQUEST_STRATEGY),
145 ds.getPrimitives(), cs, NullProgressMonitor.INSTANCE);
146 OsmApi.getOsmApi().closeChangeset(cs, NullProgressMonitor.INSTANCE);
147 }
148
149 static DataSet testDataSet;
150
151 /**
152 * Setup test.
153 * @throws OsmTransferException if something goes wrong
154 * @throws CyclicUploadDependencyException if a cyclic dependency is detected
155 * @throws IOException if an I/O error occurs
156 */
157 @BeforeClass
158 public static void setUpBeforeClass() throws OsmTransferException, CyclicUploadDependencyException, IOException {
159 logger.info("initializing ...");
160
161 JOSMFixture.createFunctionalTestFixture().init();
162
163 Main.pref.put("osm-server.auth-method", "basic");
164
165 // don't use atomic upload, the test API server can't cope with large diff uploads
166 //
167 Main.pref.put("osm-server.atomic-upload", false);
168 Main.setProjection(Projections.getProjectionByCode("EPSG:3857")); // Mercator
169 Logging.setLogLevel(Logging.LEVEL_DEBUG);
170
171 File dataSetCacheOutputFile = new File(System.getProperty("java.io.tmpdir"),
172 MultiFetchServerObjectReaderTest.class.getName() + ".dataset");
173
174 String p = System.getProperty("useCachedDataset");
175 if (p != null && Boolean.parseBoolean(p.trim().toLowerCase(Locale.ENGLISH))) {
176 logger.info(MessageFormat.format("property ''{0}'' set, using cached dataset", "useCachedDataset"));
177 return;
178 }
179
180 logger.info(MessageFormat.format(
181 "property ''{0}'' not set to true, creating test dataset on the server. property is ''{1}''", "useCachedDataset", p));
182
183 // build and upload the test data set
184 //
185 logger.info("creating test data set ....");
186 testDataSet = buildTestDataSet();
187 logger.info("uploading test data set ...");
188 createDataSetOnServer(new APIDataSet(testDataSet));
189
190 try (
191 PrintWriter pw = new PrintWriter(
192 new OutputStreamWriter(new FileOutputStream(dataSetCacheOutputFile), StandardCharsets.UTF_8)
193 )) {
194 logger.info(MessageFormat.format("caching test data set in ''{0}'' ...", dataSetCacheOutputFile.toString()));
195 try (OsmWriter w = new OsmWriter(pw, false, testDataSet.getVersion())) {
196 w.header();
197 w.writeDataSources(testDataSet);
198 w.writeContent(testDataSet);
199 w.footer();
200 }
201 }
202 }
203
204 private DataSet ds;
205
206 /**
207 * Setup test.
208 * @throws IOException if any I/O error occurs
209 * @throws IllegalDataException if an error was found while parsing the OSM data
210 * @throws FileNotFoundException if the dataset file cannot be found
211 */
212 @Before
213 public void setUp() throws IOException, IllegalDataException, FileNotFoundException {
214 File f = new File(System.getProperty("java.io.tmpdir"), MultiFetchServerObjectReaderTest.class.getName() + ".dataset");
215 logger.info(MessageFormat.format("reading cached dataset ''{0}''", f.toString()));
216 ds = new DataSet();
217 try (FileInputStream fis = new FileInputStream(f)) {
218 ds = OsmReader.parseDataSet(fis, NullProgressMonitor.INSTANCE);
219 }
220 }
221
222 /**
223 * Test reading references for a node.
224 * @throws OsmTransferException if an error occurs
225 */
226 @Test
227 public void testBackreferenceForNode() throws OsmTransferException {
228 Node n = lookupNode(ds, 0);
229 assertNotNull(n);
230 Way w = lookupWay(ds, 0);
231 assertNotNull(w);
232
233 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(n);
234 reader.setReadFull(false);
235 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
236 printNumberOfPrimitives(referers);
237
238 Set<Long> expectedNodeIds = new HashSet<>();
239 Set<Long> expectedWayIds = new HashSet<>();
240 Set<Long> expectedRelationIds = new HashSet<>();
241
242 for (OsmPrimitive ref : n.getReferrers()) {
243 if (ref instanceof Way) {
244 expectedWayIds.add(ref.getId());
245 expectedNodeIds.addAll(getNodeIdsInWay((Way) ref));
246 } else if (ref instanceof Relation) {
247 expectedRelationIds.add(ref.getId());
248 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, false));
249 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
250 }
251 }
252
253 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
254 assertEquals(expectedWayIds.size(), referers.getWays().size());
255 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
256
257 for (Node node : referers.getNodes()) {
258 assertTrue(expectedNodeIds.contains(node.getId()));
259 assertFalse(node.isIncomplete());
260 }
261
262 for (Way way : referers.getWays()) {
263 assertTrue(expectedWayIds.contains(way.getId()));
264 assertEquals(n.getReferrers().contains(way), !way.isIncomplete());
265 }
266
267 for (Relation relation : referers.getRelations()) {
268 assertTrue(expectedRelationIds.contains(relation.getId()));
269 assertFalse(relation.isIncomplete());
270 }
271 }
272
273 private void printNumberOfPrimitives(DataSet referers) {
274 System.out.println("#nodes=" + referers.getNodes().size() +
275 " #ways=" + referers.getWays().size() +
276 " #relations=" + referers.getRelations().size());
277 }
278
279 /**
280 * Test reading full references for a node.
281 * @throws OsmTransferException if an error occurs
282 */
283 @Test
284 public void testBackreferenceForNodeFull() throws OsmTransferException {
285 Node n = lookupNode(ds, 0);
286 assertNotNull(n);
287
288 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(n);
289 reader.setReadFull(true);
290 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
291 printNumberOfPrimitives(referers);
292
293 Set<Long> expectedNodeIds = new HashSet<>();
294 Set<Long> expectedWayIds = new HashSet<>();
295 Set<Long> expectedRelationIds = new HashSet<>();
296 for (OsmPrimitive ref : n.getReferrers()) {
297 if (ref instanceof Way) {
298 expectedWayIds.add(ref.getId());
299 expectedNodeIds.addAll(getNodeIdsInWay((Way) ref));
300 } else if (ref instanceof Relation) {
301 expectedRelationIds.add(ref.getId());
302 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, true));
303 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, true));
304 }
305 }
306
307 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
308 assertEquals(expectedWayIds.size(), referers.getWays().size());
309 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
310
311 for (Node node : referers.getNodes()) {
312 assertTrue(expectedNodeIds.contains(node.getId()));
313 assertFalse(node.isIncomplete());
314 }
315
316 for (Way way : referers.getWays()) {
317 assertTrue(expectedWayIds.contains(way.getId()));
318 assertFalse(way.isIncomplete());
319 }
320
321 for (Relation relation : referers.getRelations()) {
322 assertTrue(expectedRelationIds.contains(relation.getId()));
323 assertFalse(relation.isIncomplete());
324 }
325 }
326
327 /**
328 * Test reading references for a way.
329 * @throws OsmTransferException if an error occurs
330 */
331 @Test
332 public void testBackreferenceForWay() throws OsmTransferException {
333 Way w = lookupWay(ds, 1);
334 assertNotNull(w);
335 // way with name "way-1" is referred to by two relations
336 //
337
338 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(w);
339 reader.setReadFull(false);
340 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
341 printNumberOfPrimitives(referers);
342
343 Set<Long> expectedNodeIds = new HashSet<>();
344 Set<Long> expectedWayIds = new HashSet<>();
345 Set<Long> expectedRelationIds = new HashSet<>();
346
347 for (OsmPrimitive ref : w.getReferrers()) {
348 if (ref instanceof Relation) {
349 expectedRelationIds.add(ref.getId());
350 expectedWayIds.addAll(getWayIdsInRelation((Relation) ref, false));
351 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
352 }
353 }
354
355 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
356 assertEquals(expectedWayIds.size(), referers.getWays().size());
357 assertEquals(expectedRelationIds.size(), referers.getRelations().size());
358
359 for (Way w1 : referers.getWays()) {
360 assertTrue(w1.isIncomplete());
361 }
362 assertEquals(2, referers.getRelations().size()); // two relations referring to w
363
364 Relation r = lookupRelation(referers, 0);
365 assertNotNull(r);
366 assertFalse(r.isIncomplete());
367 r = lookupRelation(referers, 1);
368 assertFalse(r.isIncomplete());
369 }
370
371 /**
372 * Test reading full references for a way.
373 * @throws OsmTransferException if an error occurs
374 */
375 @Test
376 public void testBackreferenceForWayFull() throws OsmTransferException {
377 Way w = lookupWay(ds, 1);
378 assertNotNull(w);
379 // way with name "way-1" is referred to by two relations
380 //
381
382 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(w);
383 reader.setReadFull(true);
384 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
385 assertEquals(6, referers.getWays().size()); // 6 ways referred by two relations
386 for (Way w1 : referers.getWays()) {
387 assertFalse(w1.isIncomplete());
388 }
389 assertEquals(2, referers.getRelations().size()); // two relations referring to
390 Set<Long> expectedNodeIds = new HashSet<>();
391 for (Way way : referers.getWays()) {
392 Way orig = (Way) ds.getPrimitiveById(way);
393 for (Node n : orig.getNodes()) {
394 expectedNodeIds.add(n.getId());
395 }
396 }
397 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
398 for (Node n : referers.getNodes()) {
399 assertTrue(expectedNodeIds.contains(n.getId()));
400 }
401
402 Relation r = lookupRelation(referers, 0);
403 assertNotNull(r);
404 assertFalse(r.isIncomplete());
405 r = lookupRelation(referers, 1);
406 assertFalse(r.isIncomplete());
407 }
408
409 /**
410 * Test reading references for a relation.
411 * @throws OsmTransferException if an error occurs
412 */
413 @Test
414 public void testBackreferenceForRelation() throws OsmTransferException {
415 Relation r = lookupRelation(ds, 1);
416 assertNotNull(r);
417 // way with name "relation-1" is referred to by four relations:
418 // relation-6, relation-7, relation-8, relation-9
419 //
420
421 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(r);
422 reader.setReadFull(false);
423 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
424 printNumberOfPrimitives(referers);
425
426 Set<Long> referringRelationsIds = new HashSet<>();
427 Relation r6 = lookupRelation(referers, 6);
428 assertNotNull(r6);
429 assertFalse(r6.isIncomplete());
430 referringRelationsIds.add(r6.getId());
431 Relation r7 = lookupRelation(referers, 7);
432 assertNotNull(r7);
433 assertFalse(r7.isIncomplete());
434 referringRelationsIds.add(r7.getId());
435 Relation r8 = lookupRelation(referers, 8);
436 assertNotNull(r8);
437 assertFalse(r8.isIncomplete());
438 referringRelationsIds.add(r8.getId());
439 Relation r9 = lookupRelation(referers, 9);
440 assertNotNull(r9);
441 assertFalse(r9.isIncomplete());
442 referringRelationsIds.add(r9.getId());
443
444 for (Relation r1 : referers.getRelations()) {
445 if (!referringRelationsIds.contains(r1.getId())) {
446 assertTrue(r1.isIncomplete());
447 }
448 }
449
450 // make sure we read all ways referred to by parent relations. These
451 // ways are incomplete after reading.
452 //
453 Set<Long> expectedWayIds = new HashSet<>();
454 for (RelationMember m : lookupRelation(ds, 6).getMembers()) {
455 if (m.isWay()) {
456 expectedWayIds.add(m.getMember().getId());
457 }
458 }
459 for (RelationMember m : lookupRelation(ds, 7).getMembers()) {
460 if (m.isWay()) {
461 expectedWayIds.add(m.getMember().getId());
462 }
463 }
464 for (RelationMember m : lookupRelation(ds, 8).getMembers()) {
465 if (m.isWay()) {
466 expectedWayIds.add(m.getMember().getId());
467 }
468 }
469 for (RelationMember m : lookupRelation(ds, 9).getMembers()) {
470 if (m.isWay()) {
471 expectedWayIds.add(m.getMember().getId());
472 }
473 }
474
475 assertEquals(expectedWayIds.size(), referers.getWays().size());
476 for (Way w1 : referers.getWays()) {
477 assertTrue(expectedWayIds.contains(w1.getId()));
478 assertTrue(w1.isIncomplete());
479 }
480
481 // make sure we read all nodes referred to by parent relations.
482 Set<Long> expectedNodeIds = new HashSet<>();
483 for (OsmPrimitive ref : r.getReferrers()) {
484 if (ref instanceof Relation) {
485 expectedNodeIds.addAll(getNodeIdsInRelation((Relation) ref, false));
486 }
487 }
488 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
489 }
490
491 protected static Set<Long> getNodeIdsInWay(Way way) {
492 HashSet<Long> ret = new HashSet<>();
493 if (way == null) return ret;
494 for (Node n: way.getNodes()) {
495 ret.add(n.getId());
496 }
497 return ret;
498 }
499
500 protected static Set<Long> getNodeIdsInRelation(Relation r, boolean children) {
501 HashSet<Long> ret = new HashSet<>();
502 if (r == null) return ret;
503 for (RelationMember m: r.getMembers()) {
504 if (m.isNode()) {
505 ret.add(m.getMember().getId());
506 } else if (m.isWay() && children) {
507 ret.addAll(getNodeIdsInWay(m.getWay()));
508 } else if (m.isRelation() && children) {
509 ret.addAll(getNodeIdsInRelation(m.getRelation(), true));
510 }
511 }
512 return ret;
513 }
514
515 protected static Set<Long> getWayIdsInRelation(Relation r, boolean children) {
516 HashSet<Long> ret = new HashSet<>();
517 if (r == null) return ret;
518 for (RelationMember m: r.getMembers()) {
519 if (m.isWay()) {
520 ret.add(m.getMember().getId());
521 } else if (m.isRelation() && children) {
522 ret.addAll(getWayIdsInRelation(m.getRelation(), true));
523 }
524 }
525 return ret;
526 }
527
528 /**
529 * Test reading full references for a relation.
530 * @throws OsmTransferException if an error occurs
531 */
532 @Test
533 public void testBackreferenceForRelationFull() throws OsmTransferException {
534 Relation r = lookupRelation(ds, 1);
535 assertNotNull(r);
536 // way with name "relation-1" is referred to by four relations:
537 // relation-6, relation-7, relation-8, relation-9
538 //
539
540 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(r);
541 reader.setReadFull(true);
542 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
543
544 r = lookupRelation(referers, 6);
545 assertNotNull(r);
546 assertFalse(r.isIncomplete());
547 r = lookupRelation(referers, 7);
548 assertNotNull(r);
549 assertFalse(r.isIncomplete());
550 r = lookupRelation(referers, 8);
551 assertNotNull(r);
552 assertFalse(r.isIncomplete());
553 r = lookupRelation(referers, 9);
554 assertNotNull(r);
555 assertFalse(r.isIncomplete());
556
557 // all relations are fully loaded
558 //
559 for (Relation r1 : referers.getRelations()) {
560 assertFalse(r1.isIncomplete());
561 }
562
563 // make sure we read all ways referred to by parent relations. These
564 // ways are completely read after reading the relations
565 //
566 Set<Long> expectedWayIds = new HashSet<>();
567 for (RelationMember m : lookupRelation(ds, 6).getMembers()) {
568 if (m.isWay()) {
569 expectedWayIds.add(m.getMember().getId());
570 }
571 }
572 for (RelationMember m : lookupRelation(ds, 7).getMembers()) {
573 if (m.isWay()) {
574 expectedWayIds.add(m.getMember().getId());
575 }
576 }
577 for (RelationMember m : lookupRelation(ds, 8).getMembers()) {
578 if (m.isWay()) {
579 expectedWayIds.add(m.getMember().getId());
580 }
581 }
582 for (RelationMember m : lookupRelation(ds, 9).getMembers()) {
583 if (m.isWay()) {
584 expectedWayIds.add(m.getMember().getId());
585 }
586 }
587 for (long id : expectedWayIds) {
588 Way w = (Way) referers.getPrimitiveById(id, OsmPrimitiveType.WAY);
589 assertNotNull(w);
590 assertFalse(w.isIncomplete());
591 }
592
593 Set<Long> expectedNodeIds = new HashSet<>();
594 for (int i = 6; i < 10; i++) {
595 Relation r1 = lookupRelation(ds, i);
596 expectedNodeIds.addAll(getNodeIdsInRelation(r1, true));
597 }
598
599 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
600 for (Node n : referers.getNodes()) {
601 assertTrue(expectedNodeIds.contains(n.getId()));
602 }
603 }
604}
Note: See TracBrowser for help on using the repository browser.