Index: applications/editors/josm/plugins/opendata/modules/fr.datagouvfr/src/org/openstreetmap/josm/plugins/opendata/modules/fr/datagouvfr/datasets/administration/GeoFlaHandler.java =================================================================== --- applications/editors/josm/plugins/opendata/modules/fr.datagouvfr/src/org/openstreetmap/josm/plugins/opendata/modules/fr/datagouvfr/datasets/administration/GeoFlaHandler.java (revision 30907) +++ applications/editors/josm/plugins/opendata/modules/fr.datagouvfr/src/org/openstreetmap/josm/plugins/opendata/modules/fr/datagouvfr/datasets/administration/GeoFlaHandler.java (revision 30908) @@ -8,4 +8,5 @@ import org.apache.commons.lang3.text.WordUtils; +import org.openstreetmap.josm.Main; import org.openstreetmap.josm.data.coor.EastNorth; import org.openstreetmap.josm.data.coor.LatLon; @@ -18,8 +19,16 @@ import org.openstreetmap.josm.tools.Pair; +/** + * Handler for GeoFla 2.0. Compatibility for previous version 1.1 has been dropped. + * See http://professionnels.ign.fr/sites/default/files/DC_GEOFLA_2-0.pdf + */ public class GeoFlaHandler extends DataGouvDataSetHandler { + private static final String ADMIN_LEVEL = "admin_level"; + + /** + * Constructs a new {@code GeoFlaHandler}. + */ public GeoFlaHandler() { - super(); setName("GEOFLA®"); getShpHandler().setPreferMultipolygonToSimpleWay(true); @@ -27,5 +36,5 @@ setLocalPortalURL("http://professionnels.ign.fr/geofla#tab-3"); } catch (MalformedURLException e) { - e.printStackTrace(); + Main.error(e); } } @@ -74,19 +83,21 @@ p.put("name", deptName); } else if (isCommuneFile(filename)) { - p.put("name", WordUtils.capitalizeFully(getAndRemoveIgnoreCase(p, "NOM_COMM"))); + p.put("name", WordUtils.capitalizeFully(getAndRemoveIgnoreCase(p, "NOM_COM"))); replace(p, "INSEE_COM", "ref:INSEE"); } + getAndRemoveIgnoreCase(p, "NOM_REG"); + replace(p, "POPULATION", "population"); p.put("boundary", "administrative"); String nature = getIgnoreCase(p, "Nature"); if ("Frontière internationale".equalsIgnoreCase(nature) || "Limite côtière".equalsIgnoreCase(nature)) { - p.put("admin_level", "2"); + p.put(ADMIN_LEVEL, "2"); } else if ("Limite de région".equalsIgnoreCase(nature)) { - p.put("admin_level", "4"); + p.put(ADMIN_LEVEL, "4"); } else if (isDepartementFile(filename) || "Limite de département".equalsIgnoreCase(nature)) { - p.put("admin_level", "6"); + p.put(ADMIN_LEVEL, "6"); } else if(isArrondissementFile(filename) || "Limite d'arrondissement".equalsIgnoreCase(nature)) { - p.put("admin_level", "7"); + p.put(ADMIN_LEVEL, "7"); } else if(isCommuneFile(filename)) { - p.put("admin_level", "8"); + p.put(ADMIN_LEVEL, "8"); } if (p instanceof Relation) { @@ -97,5 +108,4 @@ Node centroid = new Node(llCentroid); ds.addPrimitive(centroid); - //centroid.put("name", p.get("name")); if (p instanceof Relation) { ((Relation) p).addMember(new RelationMember("centroid", centroid)); @@ -106,10 +116,26 @@ Node chefLieu = new Node(llChefLieu); ds.addPrimitive(chefLieu); - //chefLieu.put("Code_chf", getAndRemoveIgnoreCase(p, "Code_chf", "Code_Chef_Lieu")); String name = WordUtils.capitalizeFully(getAndRemoveIgnoreCase(p, "Nom_chf", "Nom_Chef_lieu")); - if (isArrondissementFile(filename)) { - p.put("name", name); - } - chefLieu.put("name", name); + if (name != null) { + if (isArrondissementFile(filename)) { + p.put("name", name); + } + chefLieu.put("name", name); + } + String population = p.get("population"); + if (population != null) { + try { + int pop = Integer.parseInt(population); + if (pop < 2000) { + chefLieu.put("place", "village"); + } else if (pop < 100000) { + chefLieu.put("place", "town"); + } else { + chefLieu.put("place", "city"); + } + } catch (NumberFormatException e) { + Main.warn("Invalid population: "+population); + } + } if (p instanceof Relation) { ((Relation) p).addMember(new RelationMember("admin_centre", chefLieu)); @@ -166,10 +192,10 @@ dptCode = "976"; } else { - System.err.println("Unknown French department: "+dptName); - } - } - return getLatLonByDptCode(new EastNorth(Double.parseDouble(x)*100.0, Double.parseDouble(y)*100.0), dptCode, false); + Main.error("Unknown French department: "+dptName); + } + } + return getLatLonByDptCode(new EastNorth(Double.parseDouble(x), Double.parseDouble(y)), dptCode, false); } catch (NumberFormatException e) { - System.err.println(e.getMessage()); + Main.error(e); } } @@ -178,5 +204,5 @@ private Pair getGeoflaURL(String name, String urlSuffix) throws MalformedURLException { - return new Pair<>(name, new URL("http://professionnels.ign.fr/sites/default/files/"+urlSuffix)); + return new Pair<>(name, new URL("https://wxs-telechargement.ign.fr/oikr5jryiph0iwhw36053ptm/telechargement/inspire/"+urlSuffix)); } @@ -185,15 +211,33 @@ List> result = new ArrayList<>(); try { - result.add(getGeoflaURL("Départements France métropolitaine et Corse", "GEOFLADept_FR_Corse_AV_L93.zip")); - result.add(getGeoflaURL("Départements France entière", "FR_DOM_Mayotte_shp_WGS84.zip")); - // FIXME: tar.gz files - /*result.add(getGeoflaURL("Communes France métropolitaine", "531/266/5312664/GEOFLA_1-1_SHP_LAMB93_FR-ED111.tar.gz")); - result.add(getGeoflaURL("Communes Guadeloupe", "531/265/5312650/GEOFLA_1-1_SHP_UTM20W84_GP-ED111.tar.gz")); - result.add(getGeoflaURL("Communes Martinique", "531/265/5312653/GEOFLA_1-1_SHP_UTM20W84_MQ-ED111.tar.gz")); - result.add(getGeoflaURL("Communes Guyane", "531/265/5312657/GEOFLA_1-1_SHP_UTM22RGFG95_GF-ED111.tar.gz")); - result.add(getGeoflaURL("Communes Réunion", "531/266/5312660/GEOFLA_1-1_SHP_RGR92UTM40S_RE-ED111.tar.gz")); - result.add(getGeoflaURL("Communes Mayotte", "531/275/5312753/GEOFLA_1-1_SHP_RGM04UTM38S_YT-ED111.tar.gz"));*/ + // Communes + result.add(getGeoflaURL("2014 Communes France Métropolitaine", "GEOFLA_THEME-COMMUNE_2014_GEOFLA_2-0_COMMUNE_SHP_LAMB93_FXX_2014-12-05/file/GEOFLA_2-0_COMMUNE_SHP_LAMB93_FXX_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Communes Guadeloupe", "GEOFLA_THEME-COMMUNE_2014_GEOFLA_2-0_COMMUNE_SHP_UTM20W84GUAD_D971_2014-12-08/file/GEOFLA_2-0_COMMUNE_SHP_UTM20W84GUAD_D971_2014-12-08.7z")); + result.add(getGeoflaURL("2014 Communes Martinique", "GEOFLA_THEME-COMMUNE_2014_GEOFLA_2-0_COMMUNE_SHP_UTM20W84MART_D972_2014-12-08/file/GEOFLA_2-0_COMMUNE_SHP_UTM20W84MART_D972_2014-12-08.7z")); + result.add(getGeoflaURL("2014 Communes Guyane", "GEOFLA_THEME-COMMUNE_2014_GEOFLA_2-0_COMMUNE_SHP_UTM22RGFG95_D973_2014-12-05/file/GEOFLA_2-0_COMMUNE_SHP_UTM22RGFG95_D973_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Communes Réunion", "GEOFLA_THEME-COMMUNE_2014_GEOFLA_2-0_COMMUNE_SHP_RGR92UTM40S_D974_2014-12-05/file/GEOFLA_2-0_COMMUNE_SHP_RGR92UTM40S_D974_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Communes Mayotte", "GEOFLA_THEME-COMMUNE_2014_GEOFLA_2-0_COMMUNE_SHP_RGM04UTM38S_D976_2014-12-05/file/GEOFLA_2-0_COMMUNE_SHP_RGM04UTM38S_D976_2014-12-05.7z")); + // Cantons + result.add(getGeoflaURL("2014 Cantons France Métropolitaine", "GEOFLA_THEME-CANTON_2014_GEOFLA_2-0_CANTON_SHP_LAMB93_FXX_2014-12-05/file/GEOFLA_2-0_CANTON_SHP_LAMB93_FXX_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Cantons Guadeloupe", "GEOFLA_THEME-CANTON_2014_GEOFLA_2-0_CANTON_SHP_UTM20W84GUAD_D971_2014-12-08/file/GEOFLA_2-0_CANTON_SHP_UTM20W84GUAD_D971_2014-12-08.7z")); + result.add(getGeoflaURL("2014 Cantons Martinique", "GEOFLA_THEME-CANTON_2014_GEOFLA_2-0_CANTON_SHP_UTM20W84MART_D972_2014-12-08/file/GEOFLA_2-0_CANTON_SHP_UTM20W84MART_D972_2014-12-08.7z")); + result.add(getGeoflaURL("2014 Cantons Guyane", "GEOFLA_THEME-CANTON_2014_GEOFLA_2-0_CANTON_SHP_UTM22RGFG95_D973_2014-12-05/file/GEOFLA_2-0_CANTON_SHP_UTM22RGFG95_D973_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Cantons Réunion", "GEOFLA_THEME-CANTON_2014_GEOFLA_2-0_CANTON_SHP_RGR92UTM40S_D974_2014-12-05/file/GEOFLA_2-0_CANTON_SHP_RGR92UTM40S_D974_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Cantons Mayotte", "GEOFLA_THEME-CANTON_2014_GEOFLA_2-0_CANTON_SHP_RGM04UTM38S_D976_2014-12-05/file/GEOFLA_2-0_CANTON_SHP_RGM04UTM38S_D976_2014-12-05.7z")); + // Arrondissements + result.add(getGeoflaURL("2014 Arrondissements France Métropolitaine", "GEOFLA_THEME-ARRONDISSEMENT_2014_GEOFLA_2-0_ARRONDISSEMENT_SHP_LAMB93_FXX_2014-12-05/file/GEOFLA_2-0_ARRONDISSEMENT_SHP_LAMB93_FXX_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Arrondissements Guadeloupe", "GEOFLA_THEME-ARRONDISSEMENT_2014_GEOFLA_2-0_ARRONDISSEMENT_SHP_UTM20W84GUAD_D971_2014-12-08/file/GEOFLA_2-0_ARRONDISSEMENT_SHP_UTM20W84GUAD_D971_2014-12-08.7z")); + result.add(getGeoflaURL("2014 Arrondissements Martinique", "GEOFLA_THEME-ARRONDISSEMENT_2014_GEOFLA_2-0_ARRONDISSEMENT_SHP_UTM20W84MART_D972_2014-12-08/file/GEOFLA_2-0_ARRONDISSEMENT_SHP_UTM20W84MART_D972_2014-12-08.7z")); + result.add(getGeoflaURL("2014 Arrondissements Guyane", "GEOFLA_THEME-ARRONDISSEMENT_2014_GEOFLA_2-0_ARRONDISSEMENT_SHP_UTM22RGFG95_D973_2014-12-05/file/GEOFLA_2-0_ARRONDISSEMENT_SHP_UTM22RGFG95_D973_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Arrondissements Réunion", "GEOFLA_THEME-ARRONDISSEMENT_2014_GEOFLA_2-0_ARRONDISSEMENT_SHP_RGR92UTM40S_D974_2014-12-05/file/GEOFLA_2-0_ARRONDISSEMENT_SHP_RGR92UTM40S_D974_2014-12-05.7z")); + // Départements + result.add(getGeoflaURL("2014 Départements France Métropolitaine", "GEOFLA_THEME-DEPARTEMENTS_2014_GEOFLA_2-0_DEPARTEMENT_SHP_LAMB93_FXX_2014-12-05/file/GEOFLA_2-0_DEPARTEMENT_SHP_LAMB93_FXX_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Départements Guadeloupe", "GEOFLA_THEME-DEPARTEMENTS_2014_GEOFLA_2-0_DEPARTEMENT_SHP_UTM20W84GUAD_D971_2014-12-08/file/GEOFLA_2-0_DEPARTEMENT_SHP_UTM20W84GUAD_D971_2014-12-08.7z")); + result.add(getGeoflaURL("2014 Départements Martinique", "GEOFLA_THEME-DEPARTEMENTS_2014_GEOFLA_2-0_DEPARTEMENT_SHP_UTM20W84MART_D972_2014-12-08/file/GEOFLA_2-0_DEPARTEMENT_SHP_UTM20W84MART_D972_2014-12-08.7z")); + result.add(getGeoflaURL("2014 Départements Guyane", "GEOFLA_THEME-DEPARTEMENTS_2014_GEOFLA_2-0_DEPARTEMENT_SHP_UTM22RGFG95_D973_2014-12-05/file/GEOFLA_2-0_DEPARTEMENT_SHP_UTM22RGFG95_D973_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Départements Réunion", "GEOFLA_THEME-DEPARTEMENTS_2014_GEOFLA_2-0_DEPARTEMENT_SHP_RGR92UTM40S_D974_2014-12-05/file/GEOFLA_2-0_DEPARTEMENT_SHP_RGR92UTM40S_D974_2014-12-05.7z")); + result.add(getGeoflaURL("2014 Départements Mayotte", "GEOFLA_THEME-DEPARTEMENTS_2014_GEOFLA_2-0_DEPARTEMENT_SHP_RGM04UTM38S_D976_2014-12-05/file/GEOFLA_2-0_DEPARTEMENT_SHP_RGM04UTM38S_D976_2014-12-05.7z")); } catch (MalformedURLException e) { - e.printStackTrace(); + Main.error(e); } return result; Index: applications/editors/josm/plugins/opendata/src/org/openstreetmap/josm/plugins/opendata/core/io/archive/SevenZipReader.java =================================================================== --- applications/editors/josm/plugins/opendata/src/org/openstreetmap/josm/plugins/opendata/core/io/archive/SevenZipReader.java (revision 30907) +++ applications/editors/josm/plugins/opendata/src/org/openstreetmap/josm/plugins/opendata/core/io/archive/SevenZipReader.java (revision 30908) @@ -33,5 +33,5 @@ private final IInArchive archive = new Handler(); - + public SevenZipReader(InputStream in, AbstractDataSetHandler handler, boolean promptUser) throws IOException { super(handler, handler != null ? handler.getArchiveHandler() : null, promptUser); @@ -41,22 +41,24 @@ Utils.copyStream(in, out); } - try (IInStream random = new MyRandomAccessFile(tmpFile.getPath(), "r")) { - if (archive.Open(random) != 0) { - String message = "Unable to open 7z archive: "+tmpFile.getPath(); - Main.warn(message); - if (!tmpFile.delete()) { - tmpFile.deleteOnExit(); - } - throw new IOException(message); + // random must be kept open for later extracting + @SuppressWarnings("resource") + IInStream random = new MyRandomAccessFile(tmpFile.getPath(), "r"); + if (archive.Open(random) != 0) { + String message = "Unable to open 7z archive: "+tmpFile.getPath(); + Main.warn(message); + random.close(); + if (!tmpFile.delete()) { + tmpFile.deleteOnExit(); } + throw new IOException(message); } } - - public static DataSet parseDataSet(InputStream in, AbstractDataSetHandler handler, ProgressMonitor instance, boolean promptUser) + + public static DataSet parseDataSet(InputStream in, AbstractDataSetHandler handler, ProgressMonitor instance, boolean promptUser) throws IOException, XMLStreamException, FactoryConfigurationError, JAXBException { return new SevenZipReader(in, handler, promptUser).parseDoc(instance); } - public static Map parseDataSets(InputStream in, AbstractDataSetHandler handler, ProgressMonitor instance, boolean promptUser) + public static Map parseDataSets(InputStream in, AbstractDataSetHandler handler, ProgressMonitor instance, boolean promptUser) throws IOException, XMLStreamException, FactoryConfigurationError, JAXBException { return new SevenZipReader(in, handler, promptUser).parseDocs(instance); @@ -68,11 +70,13 @@ } - @Override protected void extractArchive(File temp, List candidates) throws IOException, FileNotFoundException { + @Override + protected void extractArchive(File temp, List candidates) throws IOException, FileNotFoundException { archive.Extract(null, -1, IInArchive.NExtract_NAskMode_kExtract, new ExtractCallback(archive, temp, candidates)); + archive.close(); } - + private class ExtractCallback extends ArchiveExtractCallback { private final List candidates; - + public ExtractCallback(IInArchive archive, File tempDir, List candidates) { Init(archive); @@ -81,5 +85,5 @@ } - @Override + @Override public int GetStream(int index, OutputStream[] outStream, int askExtractMode) throws IOException { int res = super.GetStream(index, outStream, askExtractMode);