Index: applications/editors/josm/plugins/build.xml =================================================================== --- applications/editors/josm/plugins/build.xml (revision 36111) +++ applications/editors/josm/plugins/build.xml (revision 36112) @@ -13,4 +13,5 @@ + + + + + Index: applications/editors/josm/plugins/pmtiles/.project =================================================================== --- applications/editors/josm/plugins/pmtiles/.project (revision 36112) +++ applications/editors/josm/plugins/pmtiles/.project (revision 36112) @@ -0,0 +1,19 @@ + + + JOSM-pmtiles + + + JOSM + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + net.sf.eclipsecs.core.CheckstyleNature + + Index: applications/editors/josm/plugins/pmtiles/LICENSE =================================================================== --- applications/editors/josm/plugins/pmtiles/LICENSE (revision 36112) +++ applications/editors/josm/plugins/pmtiles/LICENSE (revision 36112) @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. Index: applications/editors/josm/plugins/pmtiles/README.md =================================================================== --- applications/editors/josm/plugins/pmtiles/README.md (revision 36112) +++ applications/editors/josm/plugins/pmtiles/README.md (revision 36112) @@ -0,0 +1,10 @@ +README +====== +## License +This plugin is under the GPLv2 or any later version. + +## Authors +* Taylor Smock (taylor.smock) + +## pmtiles support +At this time, this plugin only supports PMTiles v3. Index: applications/editors/josm/plugins/pmtiles/build.xml =================================================================== --- applications/editors/josm/plugins/pmtiles/build.xml (revision 36112) +++ applications/editors/josm/plugins/pmtiles/build.xml (revision 36112) @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Index: applications/editors/josm/plugins/pmtiles/org.openstreetmap.josm.plugins.pmtiles.iml =================================================================== --- applications/editors/josm/plugins/pmtiles/org.openstreetmap.josm.plugins.pmtiles.iml (revision 36112) +++ applications/editors/josm/plugins/pmtiles/org.openstreetmap.josm.plugins.pmtiles.iml (revision 36112) @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/PMTilesPlugin.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/PMTilesPlugin.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/PMTilesPlugin.java (revision 36112) @@ -0,0 +1,26 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles; + +import org.openstreetmap.josm.actions.ExtensionFileFilter; +import org.openstreetmap.josm.gui.MainApplication; +import org.openstreetmap.josm.plugins.Plugin; +import org.openstreetmap.josm.plugins.PluginInformation; +import org.openstreetmap.josm.plugins.pmtiles.actions.downloadtasks.DownloadPMTilesTask; +import org.openstreetmap.josm.plugins.pmtiles.gui.io.importexport.PMTilesFileImporter; + +/** + * This is the plugin entrypoint + */ +public final class PMTilesPlugin extends Plugin { + /** + * Creates the plugin + * + * @param info the plugin information describing the plugin. + */ + public PMTilesPlugin(PluginInformation info) { + super(info); + ExtensionFileFilter.addImporter(new PMTilesFileImporter()); + ExtensionFileFilter.updateAllFormatsImporter(); + MainApplication.getMenu().openLocation.addDownloadTaskClass(DownloadPMTilesTask.class); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/actions/downloadtasks/DownloadPMTilesTask.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/actions/downloadtasks/DownloadPMTilesTask.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/actions/downloadtasks/DownloadPMTilesTask.java (revision 36112) @@ -0,0 +1,125 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.actions.downloadtasks; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Future; + +import org.openstreetmap.josm.actions.downloadtasks.DownloadParams; +import org.openstreetmap.josm.actions.downloadtasks.DownloadTask; +import org.openstreetmap.josm.data.Bounds; +import org.openstreetmap.josm.gui.MainApplication; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.io.XmlWriter; +import org.openstreetmap.josm.plugins.pmtiles.data.imagery.PMTilesImageryInfo; +import org.openstreetmap.josm.plugins.pmtiles.gui.layers.PMTilesImageLayer; +import org.openstreetmap.josm.plugins.pmtiles.gui.layers.PMTilesMVTLayer; +import org.openstreetmap.josm.plugins.pmtiles.lib.PMTiles; +import org.openstreetmap.josm.plugins.pmtiles.lib.TileType; + +/** + * "Download" a PMTiles file. Really, this just adds a PMTiles layer. + */ +public class DownloadPMTilesTask implements DownloadTask { + private boolean zoomAfterDownload; + private boolean cancel; + private String url; + private final List errorObjects = new ArrayList<>(); + private Bounds bounds; + + private void addLayer() { + if (this.cancel) { + return; + } + try { + final var header = PMTiles.readHeader(URI.create(this.url)); + final var info = new PMTilesImageryInfo(header); + if (header.tileType() == TileType.MVT) { + MainApplication.getLayerManager().addLayer(new PMTilesMVTLayer(info)); + } else { + MainApplication.getLayerManager().addLayer(new PMTilesImageLayer(info)); + } + } catch (IOException e) { + this.errorObjects.add(e); + } + + if (this.zoomAfterDownload && this.bounds != null && MainApplication.getMap() != null && MainApplication.getMap().mapView != null) { + MainApplication.getMap().mapView.zoomTo(this.bounds); + } + } + + @Override + public Future download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { + this.bounds = downloadArea; + return MainApplication.worker.submit(this::addLayer); + } + + @Override + public Future loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) { + this.url = Objects.requireNonNull(url); + return download(settings, null, progressMonitor); + } + + @Override + public boolean acceptsUrl(String url, boolean isRemotecontrol) { + return url.endsWith(".pmtiles"); + } + + @Override + public String acceptsDocumentationSummary() { + // I think this should be a "default" implementation in DownloadTask + StringBuilder buff = new StringBuilder(128) + .append("") + .append(getTitle()) + .append(":"); + String[] patterns = getPatterns(); + if (patterns.length > 0) { + buff.append("
    "); + for (String pattern: patterns) { + buff.append("
  • ") + .append(XmlWriter.encode(pattern)) + .append("
    • "); + } + buff.append("
"); + } + buff.append(""); + return buff.toString(); + } + + @Override + public String getTitle() { + return tr("Add PMTiles layer"); + } + + @Override + public String[] getPatterns() { + return new String[]{".*.pmtiles"}; + } + + @Override + public List getErrorObjects() { + return Collections.unmodifiableList(this.errorObjects); + } + + @Override + public void cancel() { + this.cancel = true; + } + + @Override + public String getConfirmationMessage(URL url) { + return tr("Do you want to add a layer based off of {0}?", url.toExternalForm()); + } + + @Override + public void setZoomAfterDownload(boolean zoomAfterDownload) { + this.zoomAfterDownload = zoomAfterDownload; + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/data/imagery/PMTilesImageryInfo.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/data/imagery/PMTilesImageryInfo.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/data/imagery/PMTilesImageryInfo.java (revision 36112) @@ -0,0 +1,74 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.data.imagery; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Objects; + +import jakarta.json.JsonObject; +import org.openstreetmap.josm.data.imagery.ImageryInfo; +import org.openstreetmap.josm.plugins.pmtiles.lib.Header; +import org.openstreetmap.josm.plugins.pmtiles.lib.PMTiles; + +/** + * An {@link ImageryInfo} object for PMTiles + */ +public class PMTilesImageryInfo extends ImageryInfo { + private final Header header; + private final JsonObject meta; + + /** + * Create a new {@link PMTilesImageryInfo} object from a PMTiles {@link Header} + * @param header The header to use + */ + public PMTilesImageryInfo(Header header) { + Objects.requireNonNull(header); + this.setBounds(new ImageryBounds(Double.toString(header.minLatitude()) + ',' + header.minLongitude() + + ',' + header.maxLatitude() + ',' + header.maxLongitude(), ",")); + // Do NOT set the URL to the file -- the MapBoxVectorTileLayer will try to read a JSON from it (in its entirety!), + // which doesn't work well. + this.setUrl(""); + this.minZoom = header.minZoom(); + this.maxZoom = header.maxZoom(); + this.setDefaultMaxZoom(this.maxZoom); + this.setDefaultMinZoom(this.minZoom); + this.header = header; + try { + this.meta = PMTiles.readMetadata(header); + this.setName(meta.getString("name", null)); + this.description = meta.getString("description", null); + this.setAttributionText(meta.getString("attribution", null)); + if ("overlay".equals(meta.getString("type", null))) { + this.setOverlay(true); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Get the PMTiles header + * @return The header + */ + public Header header() { + return this.header; + } + + /** + * Get the PMTiles metadata + * @return The metadata + */ + public JsonObject metadata() { + return this.meta; + } + + @Override + public int hashCode() { + return 31 * super.hashCode() + this.header.hashCode(); + } + + @Override + public boolean equals(Object o) { + return super.equals(o) && this.header.equals(((PMTilesImageryInfo) o).header); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/io/importexport/PMTilesFileImporter.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/io/importexport/PMTilesFileImporter.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/io/importexport/PMTilesFileImporter.java (revision 36112) @@ -0,0 +1,40 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.io.importexport; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.File; +import java.io.IOException; + +import org.openstreetmap.josm.actions.ExtensionFileFilter; +import org.openstreetmap.josm.gui.MainApplication; +import org.openstreetmap.josm.gui.io.importexport.FileImporter; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.plugins.pmtiles.data.imagery.PMTilesImageryInfo; +import org.openstreetmap.josm.plugins.pmtiles.gui.layers.PMTilesImageLayer; +import org.openstreetmap.josm.plugins.pmtiles.gui.layers.PMTilesMVTLayer; +import org.openstreetmap.josm.plugins.pmtiles.lib.PMTiles; +import org.openstreetmap.josm.plugins.pmtiles.lib.TileType; + +/** + * Read PMTiles + */ +public class PMTilesFileImporter extends FileImporter { + /** + * Constructs a new {@link PMTilesFileImporter} + */ + public PMTilesFileImporter() { + super(new ExtensionFileFilter("pmtiles", "pmtiles", tr("PMTiles tilesets ({0})", ".pmtiles"))); + } + + @Override + public void importData(File file, ProgressMonitor progressMonitor) throws IOException { + final var header = PMTiles.readHeader(file.toURI()); + final var info = new PMTilesImageryInfo(header); + if (header.tileType() == TileType.MVT) { + MainApplication.getLayerManager().addLayer(new PMTilesMVTLayer(info)); + } else { + MainApplication.getLayerManager().addLayer(new PMTilesImageLayer(info)); + } + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/JCSCachedTileLoaderJob.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/JCSCachedTileLoaderJob.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/JCSCachedTileLoaderJob.java (revision 36112) @@ -0,0 +1,620 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.security.SecureRandom; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; + +import org.openstreetmap.josm.data.cache.CacheEntry; +import org.openstreetmap.josm.data.cache.CacheEntryAttributes; +import org.openstreetmap.josm.data.cache.ICachedLoaderJob; +import org.openstreetmap.josm.data.cache.ICachedLoaderListener; +import org.openstreetmap.josm.data.cache.ICachedLoaderListener.LoadResult; +import org.openstreetmap.josm.data.imagery.TileJobOptions; +import org.openstreetmap.josm.data.preferences.IntegerProperty; +import org.openstreetmap.josm.tools.CheckParameterUtil; +import org.openstreetmap.josm.tools.HttpClient; +import org.openstreetmap.josm.tools.Logging; +import org.openstreetmap.josm.tools.Utils; + +import org.apache.commons.jcs3.access.behavior.ICacheAccess; +import org.apache.commons.jcs3.engine.behavior.ICacheElement; + +/** + * Generic loader for HTTP based tiles. Uses custom attribute, to check, if entry has expired + * according to HTTP headers sent with tile. If so, it tries to verify using Etags + * or If-Modified-Since / Last-Modified. + *

+ * If the tile is not valid, it will try to download it from remote service and put it + * to cache. If remote server will fail it will try to use stale entry. + *

+ * This class will keep only one Job running for specified tile. All others will just finish, but + * listeners will be gathered and notified, once download job will be finished + * + * @author Wiktor Niesiobędzki + * @param cache entry key type + * @param cache value type + * @since 8168 (in JOSM). Copied to PMTilesPlugin to make some methods overridable. Methods modified are annotated with + * {@link #ModifiedFromJosm} + */ +public abstract class JCSCachedTileLoaderJob implements ICachedLoaderJob { + @Documented + @Target({ElementType.METHOD, ElementType.FIELD}) + @interface ModifiedFromJosm { + /** + * What changed from JOSM + * @return The reason or changes + */ + String value() default ""; + } + protected static final long DEFAULT_EXPIRE_TIME = TimeUnit.DAYS.toMillis(7); + // Limit for the max-age value send by the server. + protected static final long EXPIRE_TIME_SERVER_LIMIT = TimeUnit.DAYS.toMillis(28); + // Absolute expire time limit. Cached tiles that are older will not be used, + // even if the refresh from the server fails. + protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = TimeUnit.DAYS.toMillis(365); + + /** + * maximum download threads that will be started + */ + public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("cache.jcs.max_threads", 10); + + /* + * ThreadPoolExecutor starts new threads, until THREAD_LIMIT is reached. Then it puts tasks into LinkedBlockingDeque. + * + * The queue works FIFO, so one needs to take care about ordering of the entries submitted + * + * There is no point in canceling tasks, that are already taken by worker threads (if we made so much effort, we can at least cache + * the response, so later it could be used). We could actually cancel what is in LIFOQueue, but this is a tradeoff between simplicity + * and performance (we do want to have something to offer to worker threads before tasks will be resubmitted by class consumer) + */ + + private static final ThreadPoolExecutor DEFAULT_DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor( + 1, // we have a small queue, so threads will be quickly started (threads are started only, when queue is full) + THREAD_LIMIT.get(), // do not this number of threads + 30, // keepalive for thread + TimeUnit.SECONDS, + // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see) + new LinkedBlockingDeque<>(), + Utils.newThreadFactory("JCS-downloader-%d", Thread.NORM_PRIORITY) + ); + + private static final ConcurrentMap> inProgress = new ConcurrentHashMap<>(); + private static final ConcurrentMap useHead = new ConcurrentHashMap<>(); + + protected final long now; // when the job started + + @ModifiedFromJosm("Visibility") + protected final ICacheAccess cache; + private ICacheElement cacheElement; + protected V cacheData; + protected CacheEntryAttributes attributes; + + // HTTP connection parameters + private final int connectTimeout; + private final int readTimeout; + private final Map headers; + private final ThreadPoolExecutor downloadJobExecutor; + private Runnable finishTask; + private boolean force; + private final long minimumExpiryTime; + + /** + * @param cache cache instance that we will work on + * @param options options of the request + * @param downloadJobExecutor that will be executing the jobs + */ + protected JCSCachedTileLoaderJob(ICacheAccess cache, + TileJobOptions options, + ThreadPoolExecutor downloadJobExecutor) { + CheckParameterUtil.ensureParameterNotNull(cache, "cache"); + this.cache = cache; + this.now = System.currentTimeMillis(); + this.connectTimeout = options.getConnectionTimeout(); + this.readTimeout = options.getReadTimeout(); + this.headers = options.getHeaders(); + this.downloadJobExecutor = downloadJobExecutor; + this.minimumExpiryTime = TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()); + } + + /** + * @param cache cache instance that we will work on + * @param options of the request + */ + protected JCSCachedTileLoaderJob(ICacheAccess cache, + TileJobOptions options) { + this(cache, options, DEFAULT_DOWNLOAD_JOB_DISPATCHER); + } + + private void ensureCacheElement() { + if (cacheElement == null && getCacheKey() != null) { + cacheElement = cache.getCacheElement(getCacheKey()); + if (cacheElement != null) { + attributes = (CacheEntryAttributes) cacheElement.getElementAttributes(); + cacheData = cacheElement.getVal(); + } + } + } + + @Override + public V get() { + ensureCacheElement(); + return cacheData; + } + + @Override + public void submit(ICachedLoaderListener listener, boolean force) throws IOException { + this.force = force; + boolean first = false; + URL url = getUrl(); + String deduplicationKey = null; + if (url != null) { + // url might be null, for example when Bing Attribution is not loaded yet + deduplicationKey = url.toString(); + } + if (deduplicationKey == null) { + Logging.warn("No url returned for: {0}, skipping", getCacheKey()); + throw new IllegalArgumentException("No url returned"); + } + synchronized (this) { + first = !inProgress.containsKey(deduplicationKey); + } + inProgress.computeIfAbsent(deduplicationKey, k -> ConcurrentHashMap.newKeySet()).add(listener); + + if (first || force) { + // submit all jobs to separate thread, so calling thread is not blocked with IO when loading from disk + Logging.debug("JCS - Submitting job for execution for url: {0}", getUrlNoException()); + downloadJobExecutor.execute(this); + } + } + + /** + * This method is run when job has finished + */ + protected void executionFinished() { + if (finishTask != null) { + finishTask.run(); + } + } + + /** + * Checks if object from cache has sufficient data to be returned. + * @return {@code true} if object from cache has sufficient data to be returned + */ + protected boolean isObjectLoadable() { + if (cacheData == null) { + return false; + } + return cacheData.getContent().length > 0; + } + + /** + * Simple implementation. All errors should be cached as empty. Though some JDK (JDK8 on Windows for example) + * doesn't return 4xx error codes, instead they do throw an FileNotFoundException or IOException + * @param headerFields headers sent by server + * @param responseCode http status code + * + * @return true if we should put empty object into cache, regardless of what remote resource has returned + */ + protected boolean cacheAsEmpty(Map> headerFields, int responseCode) { + return attributes.getResponseCode() < 500; + } + + /** + * Returns key under which discovered server settings will be kept. + * @return key under which discovered server settings will be kept + */ + protected String getServerKey() { + try { + return getUrl().getHost(); + } catch (IOException e) { + Logging.trace(e); + return null; + } + } + + @Override + public void run() { + final Thread currentThread = Thread.currentThread(); + final String oldName = currentThread.getName(); + currentThread.setName("JCS Downloading: " + getUrlNoException()); + Logging.debug("JCS - starting fetch of url: {0} ", getUrlNoException()); + ensureCacheElement(); + try { + // try to fetch from cache + if (!force && cacheElement != null && isCacheElementValid() && isObjectLoadable()) { + // we got something in cache, and it's valid, so lets return it + Logging.debug("JCS - Returning object from cache: {0}", getCacheKey()); + finishLoading(LoadResult.SUCCESS); + return; + } + + // try to load object from remote resource + if (loadObject()) { + finishLoading(LoadResult.SUCCESS); + } else { + // if loading failed - check if we can return stale entry + if (isObjectLoadable()) { + // try to get stale entry in cache + finishLoading(LoadResult.SUCCESS); + Logging.debug("JCS - found stale object in cache: {0}", getUrlNoException()); + } else { + // failed completely + finishLoading(LoadResult.FAILURE); + } + } + } finally { + executionFinished(); + currentThread.setName(oldName); + } + } + + private void finishLoading(LoadResult result) { + Set listeners; + try { + listeners = inProgress.remove(getUrl().toString()); + } catch (IOException e) { + listeners = null; + Logging.trace(e); + } + if (listeners == null) { + Logging.warn("Listener not found for URL: {0}. Listener not notified!", getUrlNoException()); + return; + } + for (ICachedLoaderListener l: listeners) { + l.loadingFinished(cacheData, attributes, result); + } + } + + protected boolean isCacheElementValid() { + long expires = attributes.getExpirationTime(); + + // check by expire date set by server + if (expires != 0L) { + // put a limit to the expire time (some servers send a value + // that is too large) + expires = Math.min(expires, attributes.getCreateTime() + Math.max(EXPIRE_TIME_SERVER_LIMIT, minimumExpiryTime)); + if (now > expires) { + Logging.debug("JCS - Object {0} has expired -> valid to {1}, now is: {2}", + getUrlNoException(), Long.toString(expires), Long.toString(now)); + return false; + } + } else if (attributes.getLastModification() > 0 && + now - attributes.getLastModification() > Math.max(DEFAULT_EXPIRE_TIME, minimumExpiryTime)) { + // check by file modification date + Logging.debug("JCS - Object has expired, maximum file age reached {0}", getUrlNoException()); + return false; + } else if (now - attributes.getCreateTime() > Math.max(DEFAULT_EXPIRE_TIME, minimumExpiryTime)) { + Logging.debug("JCS - Object has expired, maximum time since object creation reached {0}", getUrlNoException()); + return false; + } + return true; + } + + /** + * @return true if object was successfully downloaded, false, if there was a loading failure + */ + @ModifiedFromJosm("visibility") + protected boolean loadObject() { + if (attributes == null) { + attributes = new CacheEntryAttributes(); + } + final URL url = this.getUrlNoException(); + if (url == null) { + return false; + } + + if (url.getProtocol().contains("http")) { + return loadObjectHttp(); + } + if (url.getProtocol().contains("file")) { + return loadObjectFile(url); + } + + return false; + } + + private boolean loadObjectFile(URL url) { + String fileName = url.toExternalForm(); + File file = new File(fileName.substring("file:/".length() - 1)); + if (!file.exists()) { + file = new File(fileName.substring("file://".length() - 1)); + } + try (InputStream fileInputStream = Files.newInputStream(file.toPath())) { + cacheData = createCacheEntry(Utils.readBytesFromStream(fileInputStream)); + cache.put(getCacheKey(), cacheData, attributes); + return true; + } catch (IOException e) { + Logging.error(e); + attributes.setError(e); + attributes.setException(e); + } + return false; + } + + /** + * @return true if object was successfully downloaded via http, false, if there was a loading failure + */ + private boolean loadObjectHttp() { + try { + // if we have object in cache, and host doesn't support If-Modified-Since nor If-None-Match + // then just use HEAD request and check returned values + if (isObjectLoadable() && + Boolean.TRUE.equals(useHead.get(getServerKey())) && + isCacheValidUsingHead()) { + Logging.debug("JCS - cache entry verified using HEAD request: {0}", getUrl()); + return true; + } + + Logging.debug("JCS - starting HttpClient GET request for URL: {0}", getUrl()); + final HttpClient request = getRequest("GET"); + + if (isObjectLoadable() && + (now - attributes.getLastModification()) <= ABSOLUTE_EXPIRE_TIME_LIMIT) { + request.setIfModifiedSince(attributes.getLastModification()); + } + if (isObjectLoadable() && attributes.getEtag() != null) { + request.setHeader("If-None-Match", attributes.getEtag()); + } + + final HttpClient.Response urlConn = request.connect(); + + if (urlConn.getResponseCode() == 304) { + // If isModifiedSince or If-None-Match has been set + // and the server answers with a HTTP 304 = "Not Modified" + Logging.debug("JCS - If-Modified-Since/ETag test: local version is up to date: {0}", getUrl()); + // update cache attributes + attributes = parseHeaders(urlConn); + cache.put(getCacheKey(), cacheData, attributes); + return true; + } else if (isObjectLoadable() // we have an object in cache, but we haven't received 304 response code + && ( + (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getHeaderField("ETag"))) || + attributes.getLastModification() == urlConn.getLastModified()) + ) { + // we sent ETag or If-Modified-Since, but didn't get 304 response code + // for further requests - use HEAD + String serverKey = getServerKey(); + Logging.info("JCS - Host: {0} found not to return 304 codes for If-Modified-Since or If-None-Match headers", + serverKey); + useHead.put(serverKey, Boolean.TRUE); + } + + attributes = parseHeaders(urlConn); + + for (int i = 0; i < 5; ++i) { + if (urlConn.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE) { + Thread.sleep(5000L+new SecureRandom().nextInt(5000)); + continue; + } + + attributes.setResponseCode(urlConn.getResponseCode()); + byte[] raw; + if (urlConn.getResponseCode() == HttpURLConnection.HTTP_OK) { + raw = Utils.readBytesFromStream(urlConn.getContent()); + } else { + raw = new byte[]{}; + try { + String data = urlConn.fetchContent(); + if (!data.isEmpty()) { + String detectErrorMessage = detectErrorMessage(data); + if (detectErrorMessage != null) { + attributes.setErrorMessage(detectErrorMessage); + } + } + } catch (IOException e) { + Logging.warn(e); + } + } + + if (isResponseLoadable(urlConn.getHeaderFields(), urlConn.getResponseCode(), raw)) { + // we need to check cacheEmpty, so for cases, when data is returned, but we want to store + // as empty (eg. empty tile images) to save some space + cacheData = createCacheEntry(raw); + cache.put(getCacheKey(), cacheData, attributes); + Logging.debug("JCS - downloaded key: {0}, length: {1}, url: {2}", + getCacheKey(), raw.length, getUrl()); + return true; + } else if (cacheAsEmpty(urlConn.getHeaderFields(), urlConn.getResponseCode())) { + cacheData = createCacheEntry(new byte[]{}); + cache.put(getCacheKey(), cacheData, attributes); + Logging.debug("JCS - Caching empty object {0}", getUrl()); + return true; + } else { + Logging.debug("JCS - failure during load - response is not loadable nor cached as empty"); + return false; + } + } + } catch (FileNotFoundException e) { + Logging.debug("JCS - Caching empty object as server returned 404 for: {0}", getUrlNoException()); + attributes.setResponseCode(404); + attributes.setError(e); + attributes.setException(e); + boolean doCache = isResponseLoadable(null, 404, null) || cacheAsEmpty(Collections.emptyMap(), 404); + if (doCache) { + cacheData = createCacheEntry(new byte[]{}); + cache.put(getCacheKey(), cacheData, attributes); + } + return doCache; + } catch (IOException e) { + Logging.debug("JCS - IOException during communication with server for: {0}", getUrlNoException()); + if (isObjectLoadable()) { + return true; + } else { + attributes.setError(e); + attributes.setException(e); + attributes.setResponseCode(599); // set dummy error code, greater than 500 so it will be not cached + return false; + } + + } catch (InterruptedException e) { + attributes.setError(e); + attributes.setException(e); + Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS - Exception during download {0}", getUrlNoException()); + Thread.currentThread().interrupt(); + } + Logging.warn("JCS - Silent failure during download: {0}", getUrlNoException()); + return false; + } + + /** + * Tries do detect an error message from given string. + * @param data string to analyze + * @return error message if detected, or null + * @since 14535 + */ + public String detectErrorMessage(String data) { + Matcher m = HttpClient.getTomcatErrorMatcher(data); + return m.matches() ? m.group(1).replace("'", "''") : null; + } + + /** + * Check if the object is loadable. This means, if the data will be parsed, and if this response + * will finish as successful retrieve. + * + * This simple implementation doesn't load empty response, nor client (4xx) and server (5xx) errors + * + * @param headerFields headers sent by server + * @param responseCode http status code + * @param raw data read from server + * @return true if object should be cached and returned to listener + */ + protected boolean isResponseLoadable(Map> headerFields, int responseCode, byte[] raw) { + return raw != null && raw.length != 0 && responseCode < 400; + } + + protected abstract V createCacheEntry(byte[] content); + + protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) { + CacheEntryAttributes ret = new CacheEntryAttributes(); + + /* + * according to https://www.ietf.org/rfc/rfc2616.txt Cache-Control takes precedence over max-age + * max-age is for private caches, s-max-age is for shared caches. We take any value that is larger + */ + Long expiration = 0L; + String cacheControl = urlConn.getHeaderField("Cache-Control"); + if (cacheControl != null) { + for (String token: cacheControl.split(",", -1)) { + try { + if (token.startsWith("max-age=")) { + expiration = Math.max(expiration, + TimeUnit.SECONDS.toMillis(Long.parseLong(token.substring("max-age=".length()))) + + System.currentTimeMillis() + ); + } + if (token.startsWith("s-max-age=")) { + expiration = Math.max(expiration, + TimeUnit.SECONDS.toMillis(Long.parseLong(token.substring("s-max-age=".length()))) + + System.currentTimeMillis() + ); + } + } catch (NumberFormatException e) { + // ignore malformed Cache-Control headers + Logging.trace(e); + } + } + } + + if (expiration.equals(0L)) { + expiration = urlConn.getExpiration(); + } + + // if nothing is found - set default + if (expiration.equals(0L)) { + expiration = System.currentTimeMillis() + DEFAULT_EXPIRE_TIME; + } + + ret.setExpirationTime(Math.max(minimumExpiryTime + System.currentTimeMillis(), expiration)); + ret.setLastModification(now); + ret.setEtag(urlConn.getHeaderField("ETag")); + + return ret; + } + + private HttpClient getRequest(String requestMethod) throws IOException { + final HttpClient urlConn = HttpClient.create(getUrl(), requestMethod); + urlConn.setAccept("text/html, image/png, image/jpeg, image/gif, */*"); + urlConn.setReadTimeout(readTimeout); // 30 seconds read timeout + urlConn.setConnectTimeout(connectTimeout); + if (headers != null) { + urlConn.setHeaders(headers); + } + + final boolean noCache = force + // To remove when switching to Java 11 + // Workaround for https://bugs.openjdk.java.net/browse/JDK-8146450 + || (Utils.getJavaVersion() == 8 && Utils.isRunningJavaWebStart()); + urlConn.useCache(!noCache); + + return urlConn; + } + + private boolean isCacheValidUsingHead() throws IOException { + final HttpClient.Response urlConn = getRequest("HEAD").connect(); + long lastModified = urlConn.getLastModified(); + boolean ret = (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getHeaderField("ETag"))) || + (lastModified != 0 && lastModified <= attributes.getLastModification()); + if (ret) { + // update attributes + attributes = parseHeaders(urlConn); + cache.put(getCacheKey(), cacheData, attributes); + } + return ret; + } + + /** + * TODO: move to JobFactory + * cancels all outstanding tasks in the queue. + */ + public void cancelOutstandingTasks() { + for (Runnable r: downloadJobExecutor.getQueue()) { + if (downloadJobExecutor.remove(r) && r instanceof org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob) { + ((org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob) r).handleJobCancellation(); + } + } + } + + /** + * Sets a job, that will be run, when job will finish execution + * @param runnable that will be executed + */ + public void setFinishedTask(Runnable runnable) { + this.finishTask = runnable; + + } + + /** + * Marks this job as canceled + */ + public void handleJobCancellation() { + finishLoading(LoadResult.CANCELED); + } + + private URL getUrlNoException() { + try { + return getUrl(); + } catch (IOException e) { + Logging.trace(e); + return null; + } + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTileJob.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTileJob.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTileJob.java (revision 36112) @@ -0,0 +1,139 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.jcs3.access.behavior.ICacheAccess; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; +import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; +import org.openstreetmap.josm.data.cache.CacheEntry; +import org.openstreetmap.josm.data.cache.CacheEntryAttributes; +import org.openstreetmap.josm.data.cache.ICachedLoaderListener; +import org.openstreetmap.josm.data.imagery.TileJobOptions; +import org.openstreetmap.josm.plugins.pmtiles.lib.DirectoryCache; +import org.openstreetmap.josm.plugins.pmtiles.lib.Header; +import org.openstreetmap.josm.plugins.pmtiles.lib.PMTiles; +import org.openstreetmap.josm.tools.Logging; + +/** + * A job for loading a PMTile. + */ +class PMTileJob extends JCSCachedTileLoaderJob implements TileJob, ICachedLoaderListener { + private final Tile tile; + private final Header header; + private final DirectoryCache directoryCache; + + PMTileJob(ICacheAccess cache, + TileJobOptions options, + ThreadPoolExecutor downloadJobExecutor, Header header, Tile tile, + DirectoryCache directoryCache) { + super(cache, options, downloadJobExecutor); + Objects.requireNonNull(directoryCache); + this.tile = tile; + this.header = header; + this.directoryCache = directoryCache; + } + + @Override + public void submit() { + this.submit(false); + } + + @Override + public void submit(boolean force) { + tile.initLoading(); + try { + super.submit(this, force); + } catch (IOException | IllegalArgumentException e) { + // if we fail to submit the job, mark tile as loaded and set error message + Logging.log(Logging.LEVEL_WARN, e); + tile.finishLoading(); + tile.setError(e.getMessage()); + } + } + + @Override + public String getCacheKey() { + return this.header.location().toString() + '/' + + PMTilesTileSource.getTileId(this.header, this.tile.getZoom(), this.tile.getXtile(), this.tile.getYtile()); + } + + @Override + public URL getUrl() throws IOException { + return new URL(this.getCacheKey()); + } + + @Override + public void loadingFinished(CacheEntry data, CacheEntryAttributes attributes, LoadResult result) { + switch (result) { + case FAILURE -> this.tile.setError(data == null ? + tr("Data could not be read") : new String(data.getContent(), StandardCharsets.UTF_8)); + case CANCELED -> this.tile.setLoaded(false); + case SUCCESS -> { + this.tile.finishLoading(); + this.tryLoadData(data); + } + } + } + + @Override + protected CacheEntry createCacheEntry(byte[] content) { + return switch (this.header.tileType()) { + case MVT, UNKNOWN -> new CacheEntry(content); + case JPEG, PNG, AVIF, WEBP -> new BufferedImageCacheEntry(content); + }; + } + + /** + * Try to load the tile image + * @param data The data to load + */ + private void tryLoadData(CacheEntry data) { + try (var is = new ByteArrayInputStream(data.getContent())) { + this.tile.loadImage(is); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Get the "URL" for copied methods (mostly for logging) + * @return The "URL" + */ + private String getUrlNoException() { + return getCacheKey(); + } + + /** + * @return true if object was successfully downloaded via http, false, if there was a loading failure + */ + @Override + protected boolean loadObject() { + if (attributes == null) { + attributes = new CacheEntryAttributes(); + } + try { + Logging.debug("JCS - starting HttpClient GET request for URL: {0}", getUrlNoException()); + final var data = PMTiles.readData(this.header, + PMTiles.convertToHilbert(this.tile.getZoom(), this.tile.getXtile(), this.tile.getYtile()), + this.directoryCache); + this.cacheData = this.createCacheEntry(data); + this.cache.put(getCacheKey(), this.cacheData, this.attributes); + return true; + } catch (IOException e) { + this.attributes.setError(e); + this.attributes.setException(e); + } + Logging.warn("JCS - Silent failure during download: {0}", getUrlNoException()); + return false; + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesImageLayer.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesImageLayer.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesImageLayer.java (revision 36112) @@ -0,0 +1,61 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +import java.util.Collection; +import java.util.Collections; + +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTFile; +import org.openstreetmap.josm.gui.layer.AbstractCachedTileSourceLayer; +import org.openstreetmap.josm.plugins.pmtiles.data.imagery.PMTilesImageryInfo; + +/** + * A layer for PMTiles using images + * @see PMTilesMVTLayer for MVT tiles (unfortunately it has a different inheritance tree) + */ +public class PMTilesImageLayer extends AbstractCachedTileSourceLayer implements PMTilesLayer { + /** + * Creates an instance of class based on ImageryInfo + * + * @param info ImageryInfo describing the layer + */ + public PMTilesImageLayer(PMTilesImageryInfo info) { + super(info); + if (info.getName() != null) { + this.setName(info.getName()); + } else { + this.setName(info.header().location().getPath()); + } + } + + @Override + protected Class getTileLoaderClass() { + return PMTilesLoader.class; + } + + @Override + protected void initTileSource(PMTilesImageSource tileSource) { + super.initTileSource(tileSource); + this.tileLoader = getTileLoaderFactory().makeTileLoader(this, getHeaders(tileSource), this.info.getMinimumTileExpire()); + if (this.tileLoader instanceof PMTilesLoader pmTilesLoader) { + pmTilesLoader.setInfo((PMTilesImageryInfo) this.info); + } + } + + @Override + protected String getCacheName() { + return "PMTILES_IMAGE"; + } + + @Override + public Collection getNativeProjections() { + // There is not information on what native projections are used. + // However, there are references to "Web Mercator". + return Collections.singleton(MVTFile.DEFAULT_PROJECTION); + } + + @Override + protected PMTilesImageSource getTileSource() { + return new PMTilesImageSource((PMTilesImageryInfo) this.info); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesImageSource.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesImageSource.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesImageSource.java (revision 36112) @@ -0,0 +1,48 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +import jakarta.json.JsonObject; +import org.openstreetmap.gui.jmapviewer.OsmMercator; +import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; +import org.openstreetmap.josm.plugins.pmtiles.data.imagery.PMTilesImageryInfo; +import org.openstreetmap.josm.plugins.pmtiles.lib.Header; + +/** + * A source for images in PMTiles + */ +public class PMTilesImageSource extends AbstractTMSTileSource implements PMTilesTileSource { + private final JsonObject metadata; + private final Header header; + private final OsmMercator osmMercator; + + /** + * Create a new tile source + * @param info The image raster source + */ + public PMTilesImageSource(PMTilesImageryInfo info) { + super(info); + this.metadata = info.metadata(); + this.header = info.header(); + this.osmMercator = new OsmMercator(getTileSize()); + } + + @Override + public JsonObject metadata() { + return this.metadata; + } + + @Override + public Header header() { + return this.header; + } + + @Override + public OsmMercator osmMercator() { + return this.osmMercator; + } + + @Override + public String getTileUrl(int zoom, int tilex, int tiley) { + return PMTilesTileSource.super.getTileUrl(zoom, tilex, tiley); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesLayer.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesLayer.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesLayer.java (revision 36112) @@ -0,0 +1,8 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +/** + * A common interface for layers using PMTiles as a source + */ +interface PMTilesLayer { +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesLoader.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesLoader.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesLoader.java (revision 36112) @@ -0,0 +1,72 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collection; +import java.util.HashSet; + +import org.apache.commons.jcs3.access.behavior.ICacheAccess; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; +import org.openstreetmap.josm.data.cache.CacheEntry; +import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader; +import org.openstreetmap.josm.data.imagery.TileJobOptions; +import org.openstreetmap.josm.plugins.pmtiles.data.imagery.PMTilesImageryInfo; +import org.openstreetmap.josm.plugins.pmtiles.lib.DirectoryCache; +import org.openstreetmap.josm.plugins.pmtiles.lib.Header; +import org.openstreetmap.josm.plugins.pmtiles.lib.PMTiles; + +/** + * The loader class for PMTiles + */ +public class PMTilesLoader implements TileLoader { + private final Collection jobs = new HashSet<>(); + private final ICacheAccess cache; + private final TileJobOptions options; + private final TileLoaderListener listener; + private Header header; + private DirectoryCache directoryCache; + + /** + * Create a new tile loader + * @param listener The listener to notify + * @param cache The cache to use + * @param options The options to use + */ + public PMTilesLoader(TileLoaderListener listener, ICacheAccess cache, + TileJobOptions options) { + this.listener = listener; + this.cache = cache; + this.options = options; + } + + @Override + public TileJob createTileLoaderJob(Tile tile) { + final var job = new PMTileJob(cache, options, TMSCachedTileLoader.getNewThreadPoolExecutor("pmtiles"), header, tile, directoryCache); + this.jobs.add(job); + return job; + } + + @Override + public void cancelOutstandingTasks() { + this.jobs.forEach(PMTileJob::handleJobCancellation); + this.jobs.clear(); + } + + @Override + public boolean hasOutstandingTasks() { + return this.jobs.isEmpty(); + } + + void setInfo(PMTilesImageryInfo info) { + this.header = info.header(); + try { + this.directoryCache = new DirectoryCache(PMTiles.readRootDirectory(this.header)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesMVTLayer.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesMVTLayer.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesMVTLayer.java (revision 36112) @@ -0,0 +1,62 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +import java.util.Collection; +import java.util.Collections; + +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTFile; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapboxVectorTileSource; +import org.openstreetmap.josm.gui.layer.imagery.MVTLayer; +import org.openstreetmap.josm.plugins.pmtiles.data.imagery.PMTilesImageryInfo; + +/** + * MVT layer that comes from PMTiles + * @see PMTilesImageLayer for raster tiles (unfortunately it has a different inheritance tree) + */ +public class PMTilesMVTLayer extends MVTLayer implements PMTilesLayer { + /** + * Creates an instance of an MVT layer + * + * @param info ImageryInfo describing the layer + */ + public PMTilesMVTLayer(PMTilesImageryInfo info) { + super(info); + if (info.getName() != null) { + this.setName(info.getName()); + } else { + this.setName(info.header().location().toString()); + } + } + + @Override + protected Class getTileLoaderClass() { + return PMTilesLoader.class; + } + + @Override + protected void initTileSource(MapboxVectorTileSource tileSource) { + super.initTileSource(tileSource); + this.tileLoader = getTileLoaderFactory().makeTileLoader(this, getHeaders(tileSource), this.info.getMinimumTileExpire()); + if (this.tileLoader instanceof PMTilesLoader pmTilesLoader) { + pmTilesLoader.setInfo((PMTilesImageryInfo) this.info); + } + } + + @Override + protected String getCacheName() { + return "PMTILES_IMAGE"; + } + + @Override + public Collection getNativeProjections() { + // There is not information on what native projections are used. + // However, there are references to "Web Mercator". + return Collections.singleton(MVTFile.DEFAULT_PROJECTION); + } + + @Override + protected PMTilesMVTTileSource getTileSource() { + return new PMTilesMVTTileSource((PMTilesImageryInfo) this.info); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesMVTTileSource.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesMVTTileSource.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesMVTTileSource.java (revision 36112) @@ -0,0 +1,66 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.json.JsonValue; + +import org.openstreetmap.gui.jmapviewer.OsmMercator; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapboxVectorTileSource; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.style.MapboxVectorStyle; +import org.openstreetmap.josm.plugins.pmtiles.data.imagery.PMTilesImageryInfo; +import org.openstreetmap.josm.plugins.pmtiles.lib.Header; + +/** + * The tile source for MVT tiles in PMTiles + */ +public class PMTilesMVTTileSource extends MapboxVectorTileSource implements PMTilesTileSource { + private final JsonObject metadata; + private final Header header; + private final MapboxVectorStyle styleSource; + + /** + * Create a new tile source + * @param info The MVT source + */ + public PMTilesMVTTileSource(PMTilesImageryInfo info) { + super(info); + this.metadata = info.metadata(); + this.header = info.header(); + this.baseUrl = info.header().location().toString(); + + // Check if there is a TileJSON specification file. JOSM doesn't (currently) understand it, but it may in the + // future. + if (info.metadata().containsKey("vector_layers") && info.metadata().get("vector_layers").getValueType() == JsonValue.ValueType.ARRAY) { + final var tileJson = info.metadata().getJsonArray("vector_layers"); + final var builder = Json.createObjectBuilder().add("version", 8) + .add("sources", tileJson); + if (metadata.containsKey("name")) { + builder.add("name", metadata.getString("name")); + } + this.styleSource = new MapboxVectorStyle(builder.build()); + } else { + this.styleSource = null; + } + } + + @Override + public JsonObject metadata() { + return this.metadata; + } + + @Override + public Header header() { + return this.header; + } + + @Override + public OsmMercator osmMercator() { + return this.osmMercator; + } + + @Override + public MapboxVectorStyle getStyleSource() { + return this.styleSource; + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesTileSource.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesTileSource.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/gui/layers/PMTilesTileSource.java (revision 36112) @@ -0,0 +1,281 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.gui.layers; + +import java.awt.Image; +import java.awt.Point; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import jakarta.json.JsonObject; +import org.openstreetmap.gui.jmapviewer.Coordinate; +import org.openstreetmap.gui.jmapviewer.OsmMercator; +import org.openstreetmap.gui.jmapviewer.Projected; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.TileRange; +import org.openstreetmap.gui.jmapviewer.TileXY; +import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate; +import org.openstreetmap.gui.jmapviewer.interfaces.IProjected; +import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; +import org.openstreetmap.josm.plugins.pmtiles.lib.Header; +import org.openstreetmap.josm.plugins.pmtiles.lib.PMTiles; + +/** + * The tile source for PMTiles + */ +public interface PMTilesTileSource extends TileSource { + /** + * The metdata for the source + * @return The metadata + */ + JsonObject metadata(); + + /** + * The header for the source + * @return The header + */ + Header header(); + + /** + * A mercator object for calculations + * @return The mercator object + */ + OsmMercator osmMercator(); + + @Override + default boolean requiresAttribution() { + return this.metadata().containsKey("attribution"); + } + + @Override + default String getAttributionText(int zoom, ICoordinate topLeft, ICoordinate botRight) { + return this.metadata().getString("attribution", null); + } + + @Override + default String getAttributionLinkURL() { + return null; + } + + @Override + default Image getAttributionImage() { + return null; + } + + @Override + default String getAttributionImageURL() { + return null; + } + + @Override + default String getTermsOfUseText() { + return null; + } + + @Override + default String getTermsOfUseURL() { + return null; + } + + @Override + default int getMaxZoom() { + return this.header().maxZoom(); + } + + @Override + default int getMinZoom() { + return this.header().minZoom(); + } + + @Override + default String getName() { + return this.metadata().getString("name", null); + } + + @Override + default String getId() { + return this.header().location().toString(); + } + + @Override + default String getTileUrl(int zoom, int tilex, int tiley) { + return header().location().toString() + '/' + getTileId(this.header(), zoom, tilex, tiley); + } + + @Override + default String getTileId(int zoom, int tilex, int tiley) { + return getTileId(this.header(), zoom, tilex, tiley); + } + + /** + * Get the id for this tile + * @param header The PMTile header + * @param zoom The zoom level + * @param tilex The tilex + * @param tiley The tiley + * @return The id for the tile + */ + static String getTileId(Header header, int zoom, int tilex, int tiley) { + final String extension = switch (header.tileType()) { + case MVT -> ".mvt"; + case PNG -> ".png"; + case JPEG -> ".jpg"; + case WEBP -> ".webp"; + case AVIF -> ".avif"; + case UNKNOWN -> throw new IllegalArgumentException("Unknown format: " + header.location()); + }; + return PMTiles.convertToHilbert(zoom, tilex, tiley) + extension; + } + + @Override + default int getTileSize() { + return this.getDefaultTileSize(); + } + + @Override + default int getDefaultTileSize() { + return 512; + } + + @Override + default double getDistance(double lat1, double lon1, double lat2, double lon2) { + return osmMercator().getDistance(lat1, lon1, lat2, lon2); + } + + @Override + default Point latLonToXY(double lat, double lon, int zoom) { + return new Point( + (int) Math.round(osmMercator().lonToX(lon, zoom)), + (int) Math.round(osmMercator().latToY(lat, zoom)) + ); + } + + @Override + default Point latLonToXY(ICoordinate point, int zoom) { + return this.latLonToXY(point.getLat(), point.getLon(), zoom); + } + + @Override + default ICoordinate xyToLatLon(Point point, int zoom) { + return xyToLatLon(point.x, point.y, zoom); + } + + @Override + default ICoordinate xyToLatLon(int x, int y, int zoom) { + return new Coordinate( + osmMercator().yToLat(y, zoom), + osmMercator().xToLon(x, zoom) + ); + } + + @Override + default TileXY latLonToTileXY(double lat, double lon, int zoom) { + return new TileXY( + osmMercator().lonToX(lon, zoom) / getTileSize(), + osmMercator().latToY(lat, zoom) / getTileSize() + ); + } + + @Override + default TileXY latLonToTileXY(ICoordinate point, int zoom) { + return latLonToTileXY(point.getLat(), point.getLon(), zoom); + } + + @Override + default ICoordinate tileXYToLatLon(TileXY xy, int zoom) { + return this.tileXYToLatLon(xy.getXIndex(), xy.getYIndex(), zoom); + } + + @Override + default ICoordinate tileXYToLatLon(Tile tile) { + return this.tileXYToLatLon(tile.getXtile(), tile.getYtile(), tile.getZoom()); + } + + @Override + default ICoordinate tileXYToLatLon(int x, int y, int zoom) { + return new Coordinate( + osmMercator().yToLat((long) y * getTileSize(), zoom), + osmMercator().xToLon((long) x * getTileSize(), zoom) + ); + } + + @Override + default int getTileXMax(int zoom) { + return getTileMax(zoom); + } + + @Override + default int getTileXMin(int zoom) { + return 0; + } + + @Override + default int getTileYMax(int zoom) { + return getTileMax(zoom); + } + + @Override + default int getTileYMin(int zoom) { + return 0; + } + + /** + * Get the maximum number of tiles on an axis for a specified zoom level + * @param zoom The zoom level to find the tiles for + * @return The number of tiles on an axis + */ + private static int getTileMax(int zoom) { + return (int) Math.pow(2.0, zoom) - 1; + } + + @Override + default boolean isNoTileAtZoom(Map> headers, int statusCode, byte[] content) { + return content.length == 0; + } + + @Override + default Map getMetadata(Map> headers) { + return Collections.emptyMap(); + } + + @Override + default IProjected tileXYtoProjected(int x, int y, int zoom) { + final var mercatorWidth = 2 * Math.PI * OsmMercator.EARTH_RADIUS; + final var f = mercatorWidth * getTileSize() / osmMercator().getMaxPixels(zoom); + return new Projected(f * x - mercatorWidth / 2, -(f * y - mercatorWidth / 2)); + } + + @Override + default TileXY projectedToTileXY(IProjected p, int zoom) { + final var mercatorWidth = 2 * Math.PI * OsmMercator.EARTH_RADIUS; + final var f = mercatorWidth * getTileSize() / osmMercator().getMaxPixels(zoom); + return new TileXY((p.getEast() + mercatorWidth / 2) / f, (-p.getNorth() + mercatorWidth / 2) / f); + } + + @Override + default boolean isInside(Tile inner, Tile outer) { + final int dz = inner.getZoom() - outer.getZoom(); + if (dz < 0) return false; + return outer.getXtile() == inner.getXtile() >> dz && + outer.getYtile() == inner.getYtile() >> dz; + } + + @Override + default TileRange getCoveringTileRange(Tile tile, int newZoom) { + if (newZoom <= tile.getZoom()) { + final int dz = tile.getZoom() - newZoom; + final var xy = new TileXY(tile.getXtile() >> dz, tile.getYtile() >> dz); + return new TileRange(xy, xy, newZoom); + } else { + final int dz = newZoom - tile.getZoom(); + final var t1 = new TileXY(tile.getXtile() << dz, tile.getYtile() << dz); + final var t2 = new TileXY(t1.getX() + (1 << dz) - 1, t1.getY() + (1 << dz) - 1); + return new TileRange(t1, t2, newZoom); + } + } + + @Override + default String getServerCRS() { + return "EPSG:3857"; + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/Directory.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/Directory.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/Directory.java (revision 36112) @@ -0,0 +1,17 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Iterator; + +/** + * A directory of tile/directory entries + * @param entries The entries in the directory + */ +public record Directory(DirectoryEntry... entries) implements Iterable, Serializable { + @Override + public Iterator iterator() { + return Arrays.stream(entries).iterator(); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/DirectoryCache.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/DirectoryCache.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/DirectoryCache.java (revision 36112) @@ -0,0 +1,33 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.Objects; + +/** + * A cache for directories + */ +public final class DirectoryCache implements Iterable { + private final Directory[] directories = new Directory[2]; + /** + * Create a new cache + * @param root The root directory. This is never evicted. + */ + public DirectoryCache(Directory root) { + this.directories[0] = root; + } + + /** + * Add a directory to the cache. It is highly likely to evict another directory. + * @param directory The directory to cache. + */ + public void addDirectory(Directory directory) { + this.directories[1] = directory; + } + + @Override + public Iterator iterator() { + return Arrays.stream(this.directories).filter(Objects::nonNull).iterator(); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/DirectoryEntry.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/DirectoryEntry.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/DirectoryEntry.java (revision 36112) @@ -0,0 +1,36 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +import java.io.Serializable; + +/** + * An entry for tile information + * @param tileId The tile id using Hilbert curves starting at z=0 + * @param offset The position of the file relative to the start of the data section + * @param length The size of the tile in bytes + * @param runLength The number of times the tile is repeated. 0 means that it is a leaf directory where the tileid is the first entry. + */ +public record DirectoryEntry(long tileId, long offset, long length, long runLength) implements Serializable { + public DirectoryEntry { + if (length <= 0) { + throw new IllegalArgumentException("length must be > 0"); + } + } + + /** + * Check if this entry is a leaf directory + * @return {@code true} if we need to go to a leaf directory + */ + public boolean isLeafDirectory() { + return this.runLength == 0; + } + + /** + * Check if a id is inside the range + * @param index The index to check + * @return {@code true} if this entry contains the specified index + */ + public boolean contains(long index) { + return this.tileId == index || (this.tileId < index && this.tileId + this.runLength > index); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/Header.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/Header.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/Header.java (revision 36112) @@ -0,0 +1,41 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +import java.io.Serializable; +import java.net.URI; + +/** + * The header for a PMTiles file + * @param location The location of the PMTiles. Not actually part of the PMTiles header; this is used to avoid passing URIs around. + * @param rootOffset The offset of the root directory + * @param rootLength The length of the root directory + * @param metadataOffset The offset of the metadata directory + * @param metadataLength The length of the metadata directory + * @param leafOffset The offset of leaf directories + * @param leafLength The length of leaf directories + * @param tileOffset The offset of tile data + * @param tileLength The length of tile data + * @param addressedTiles The number of addressed tiles; 0 if unknown + * @param tileEntries The number of tile entries; 0 if unknown + * @param tileContents The number of tile contents; 0 if unknown + * @param clustered {@code true} if the tiles are ordered by a Hilbert `TileId` + * @param internalCompression The compression type used for internal data + * @param tileCompression The compression type used for the tiles + * @param tileType The type of the tiles + * @param minZoom The minimum zoom level + * @param maxZoom The maximum zoom level + * @param minLongitude The minimum longitude + * @param minLatitude The minimum latitude + * @param maxLongitude The maximum longitude + * @param maxLatitude The maximum latitude + * @param centerZoom The center zoom + * @param centerLongitude The center longitude + * @param centerLatitude The center latitude + */ +public record Header(URI location, long rootOffset, long rootLength, long metadataOffset, long metadataLength, + long leafOffset, long leafLength, long tileOffset, long tileLength, long addressedTiles, + long tileEntries, long tileContents, boolean clustered, InternalCompression internalCompression, + InternalCompression tileCompression, TileType tileType, int minZoom, int maxZoom, double minLongitude, + double minLatitude, double maxLongitude, double maxLatitude, int centerZoom, double centerLongitude, + double centerLatitude) implements Serializable { +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/InternalCompression.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/InternalCompression.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/InternalCompression.java (revision 36112) @@ -0,0 +1,13 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +/** + * Internal compression details + */ +public enum InternalCompression { + UNKNOWN, + NONE, + GZIP, + BROTLI, + ZSTD +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/PMTiles.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/PMTiles.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/PMTiles.java (revision 36112) @@ -0,0 +1,289 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.InflaterInputStream; + +import jakarta.json.Json; +import jakarta.json.JsonObject; +import org.apache.commons.compress.compressors.brotli.BrotliCompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.openstreetmap.josm.plugins.pmtiles.lib.internal.DirectoryParser; +import org.openstreetmap.josm.plugins.pmtiles.lib.internal.HeaderParser; +import org.openstreetmap.josm.tools.Utils; + +/** + * The entry point for PMTiles + */ +public final class PMTiles { + private static final byte[] EMPTY_BYTE = new byte[0]; + + private PMTiles() {/* hide the constructor */} + + /** + * Read the tiles from a specified location + * @param location The location to read them from + * @return The PMTiles header + * @throws MalformedURLException if the URI could not be converted to a URL + * @throws IOException If there was an error reading the data + */ + public static Header readHeader(URI location) throws IOException { + try (var inputStream = getInputStream(location, 0, 127)) { + return HeaderParser.parse(location, inputStream); + } + } + + /** + * Read metadata from a file + * @param header The header with offset information + * @return The metadata + * @throws IOException If there was an error reading the data + */ + public static JsonObject readMetadata(Header header) throws IOException { + try (var inputStream = decompressInputStream(header.internalCompression(), + getInputStream(header.location(), header.metadataOffset(), header.metadataLength())); + var reader = Json.createReader(inputStream)) { + return reader.readObject(); + } + } + + /** + * Read the root directory + * @param header The header data + * @return The root directory + * @throws IOException If there was an error reading the data + */ + public static Directory readRootDirectory(Header header) throws IOException { + try (var inputStream = decompressInputStream(header.internalCompression(), + getInputStream(header.location(), header.rootOffset(), header.rootLength()))) { + return DirectoryParser.parse(inputStream); + } + } + + /** + * Read the root directory + * @param header The header data + * @param offset The offset inside the leaf directory area + * @param length The length of the leaf directory + * @return The root directory + * @throws IOException If there was an error reading the data + */ + public static Directory readLeafDirectory(Header header, long offset, long length) throws IOException { + try (var inputStream = decompressInputStream(header.internalCompression(), + getInputStream(header.location(), header.leafOffset() + offset, length))) { + return DirectoryParser.parse(inputStream); + } + } + + /** + * Read tile data + * @param header The header data + * @param index The hilbert index (from {@link #convertToHilbert(int, int, int)} in most cases) + * @param cachedDirectories The directories to look through. + * @return The data + * @throws IOException if the file could not be read + */ + public static byte[] readData(Header header, long index, DirectoryCache cachedDirectories) throws IOException { + final var entry = getDataLocation(header, index, cachedDirectories); + if (entry == null) { + return EMPTY_BYTE; + } + try (var inputStream = decompressInputStream(header.tileCompression(), + getInputStream(header.location(), header.tileOffset() + entry.offset(), entry.length()))) { + return inputStream.readAllBytes(); + } + } + + /** + * Get the data location in PM tiles + * @param header The header to read + * @param index The index to find + * @param cachedDirectories The directories to use to avoid recurring calls; this array should be of length 2. + * The first directory should be the root directory. The second should be the most recently used + * leaf directory, if available. + * @return The entry with the data. If {@code null} there is no entry for the data. + * @throws IOException if we could not read data + */ + public static DirectoryEntry getDataLocation(Header header, long index, DirectoryCache cachedDirectories) throws IOException { + DirectoryEntry leaf = null; + for (var directory : cachedDirectories) { + final var entry = getDataEntry(index, directory); + if (entry != null && entry.isLeafDirectory()) { + leaf = entry; + } else if (entry != null) { + return entry; + } + } + // We now need to find the appropriate leaf directory, since it wasn't in the cached directories + while (leaf != null) { + final var leafDirectory = readLeafDirectory(header, leaf.offset(), leaf.length()); + cachedDirectories.addDirectory(leafDirectory); + leaf = getDataEntry(index, leafDirectory); + if (leaf != null && leaf.contains(index) && !leaf.isLeafDirectory()) { + return leaf; + } + } + return null; + } + + /** + * Perform a search for the specified entry + * @param index The index to find + * @param directory The directory to look through + * @return The appropriate entry (may be a leaf directory or {@code null}) + */ + private static DirectoryEntry getDataEntry(long index, Directory directory) { + // This finds either the entry or the leaf directory + DirectoryEntry entry = null; + for (var current : directory) { + if (current.contains(index)) { + return current; + } else if (current.tileId() < index && current.isLeafDirectory() && (entry == null || current.tileId() > entry.tileId())) { + entry = current; + } + } + return entry; + } + + /** + * Convert a traditional tile to a Hilbert tile + * @param tile The tile to convert + * @return The Hilbert tile + */ + public static long convertToHilbert(TileXYZ tile) { + return convertToHilbert(tile.z(), tile.x(), tile.y()); + } + + /** + * Convert a traditional tile to a Hilbert tile + * @param z The z index (zoom level) + * @param x The x index + * @param y The y index + * @return The Hilbert tile + */ + public static long convertToHilbert(int z, int x, int y) { + // The maximum x/y coordinates are defined by the z level. Keep in mind that we are 0 indexed. + // 1, 4, 16, 64, ... + final var maxSquare = Math.pow(4, z); + if (x >= maxSquare || y >= maxSquare) { + throw new IllegalArgumentException("x or y out of bounds: " + z + " (x = " + x + ", y = " + y); + } + // We need to sum up the previous z levels + var start = 0; + var currentZoom = z; + // TODO profile this and the integral form (4^x)/(log(4)). Might not be as accurate though due to fp issues. + // Maybe also profile Math.pow(4, currentZoom) + while (currentZoom > 0) { + currentZoom--; + start += (1 << currentZoom) * (1 << currentZoom); + } + // Now we need to calculate the coordinates inside the specified zoom level + long d = 0; + var n = 1 << z; + final var xy = new int[]{x, y}; + for (var s = n / 2; s > 0; s /= 2) { + var rx = (xy[0] & s) > 0 ? 1 : 0; + var ry = (xy[1] & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + rotate(n, xy, rx, ry); + } + return start + d; + } + + /** + * Convert a hilbert number to a tile + * @param hilbert The continuous hilbert number + * @return The traditional tile (Z/X/Y) + */ + public static TileXYZ convertToXYZ(long hilbert) { + var z = 0; + var start = 0; + while (true) { + final var zTiles = Math.pow(4, z); + if (start + zTiles > hilbert) { + break; + } + start += zTiles; + z++; + } + long t = hilbert - start; + final var xy = new int[]{0, 0}; + final int n = 1 << z; + for (var s = 1; s < n; s *= 2) { + var rx = (int) (1 & (t / 2)); + var ry = (int) (1 & (t ^ rx)); + rotate(s, xy, rx, ry); + xy[0] += s * rx; + xy[1] += s * ry; + t /= 4; + } + return new TileXYZ(z, xy[0], xy[1]); + } + + /** + * Rotate the curve + * @param n The nxn cells + * @param xy The coordinates to modify + * @param rx The x rotation + * @param ry The y rotation + */ + private static void rotate(int n, int[] xy, int rx, int ry) { + if (ry == 0) { + if (rx == 1) { + xy[0] = n - 1 - xy[0]; + xy[1] = n - 1 - xy[1]; + } + final var temp = xy[0]; + xy[0] = xy[1]; + xy[1] = temp; + } + } + + private static InputStream decompressInputStream(InternalCompression compression, InputStream inputStream) throws IOException { + return switch (compression) { + case GZIP -> new GzipCompressorInputStream(inputStream); + case ZSTD -> new InflaterInputStream(inputStream); + case BROTLI -> new BrotliCompressorInputStream(inputStream); + case NONE -> inputStream; + case UNKNOWN -> throw new UnsupportedOperationException("Unknown compression type"); + }; + } + + private static InputStream getInputStream(URI location, long start, long length) throws IOException { + if (Utils.isLocalUrl(location.toString())) { + final var file = Path.of(location); + try (var is = Files.newInputStream(file)) { + if (start != is.skip(start)) { + throw new IOException("Something is wrong with the file"); + } + if (length < Integer.MAX_VALUE) { + return new ByteArrayInputStream(is.readNBytes((int) length)); + } else { + throw new IOException("The PMTiles plugin currently does not support large streams from the file system"); + } + } + } + var request = HttpRequest.newBuilder(location).header("Range", "bytes=" + start + "-" + (start + length)) + .header("User-Agent", "JOSM PMTiles v1").GET().build(); + try { + final var client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build(); + final var response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + if (response.statusCode() < 200 || response.statusCode() > 300) { + throw new IOException("Bad response code for " + response.request().uri() + ": " + response.statusCode()); + } + return response.body(); + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + throw new IOException(interruptedException); + } + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/TileType.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/TileType.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/TileType.java (revision 36112) @@ -0,0 +1,14 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +/** + * The type of tiles served + */ +public enum TileType { + UNKNOWN, + MVT, + PNG, + JPEG, + WEBP, + AVIF +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/TileXYZ.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/TileXYZ.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/TileXYZ.java (revision 36112) @@ -0,0 +1,11 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +/** + * A tile coordinate for web mercator + * @param x The x coordinate + * @param y The y coordinate + * @param z The z coordinate (zoom) + */ +public record TileXYZ(int z, int x, int y) { +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/DirectoryParser.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/DirectoryParser.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/DirectoryParser.java (revision 36112) @@ -0,0 +1,68 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib.internal; + +import java.io.IOException; +import java.io.InputStream; + +import org.openstreetmap.josm.plugins.pmtiles.lib.Directory; +import org.openstreetmap.josm.plugins.pmtiles.lib.DirectoryEntry; + +/** + * Parse directories from PMTiles + */ +public final class DirectoryParser { + private DirectoryParser() { /* Hide the constructor */ } + + public static Directory parse(InputStream inputStream) throws IOException { + int lastByte = inputStream.read(); + var currentInt = lastByte & 0x7F; + var shift = 7; + while ((lastByte & 0x80) != 0) { + lastByte = inputStream.read(); + currentInt |= (lastByte & 0x7F) << shift; + shift += 7; + } + final var entrySize = currentInt; + final var tileIds = new long[entrySize]; + final var runLengths = new long[entrySize]; + final var lengths = new long[entrySize]; + final var offsets = new long[entrySize]; + var index = 0; + while (lastByte != -1 && index < entrySize * 4) { + lastByte = inputStream.read(); + long currentLong = lastByte & 0x7F; + shift = 7; + while ((lastByte & 0x80) != 0 && lastByte != -1) { + lastByte = inputStream.read(); + currentLong |= (long) (lastByte & 0x7F) << shift; + shift += 7; + } + if (index < tileIds.length) { + tileIds[index] = currentLong; + } else if (index < entrySize * 2) { + runLengths[index - entrySize] = currentLong; + } else if (index < entrySize * 3) { + lengths[index - 2 * entrySize] = currentLong; + } else { + offsets[index - 3 * entrySize] = currentLong; + } + index++; + } + final var entries = new DirectoryEntry[entrySize]; + for (var i = 0; i < entries.length; i++) { + if (i == 0) { + entries[i] = new DirectoryEntry(tileIds[i], offsets[i] - 1, lengths[i], runLengths[i]); + } else { + final var lastEntry = entries[i - 1]; + final long offset; + if (offsets[i] == 0) { + offset = lastEntry.offset() + lastEntry.length(); + } else { + offset = offsets[i] - 1; + } + entries[i] = new DirectoryEntry(tileIds[i] + lastEntry.tileId(), offset, lengths[i], runLengths[i]); + } + } + return new Directory(entries); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/HeaderParser.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/HeaderParser.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/HeaderParser.java (revision 36112) @@ -0,0 +1,81 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib.internal; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; + +import org.openstreetmap.josm.plugins.pmtiles.lib.Header; +import org.openstreetmap.josm.plugins.pmtiles.lib.InternalCompression; +import org.openstreetmap.josm.plugins.pmtiles.lib.TileType; + +/** + * The implementation for headers + */ +public final class HeaderParser { + private HeaderParser() { /* Hide constructor */ } + + /** This is so that we can read the first 7 bytes and determine if we are reading a pmtiles file */ + private static final String MAGIC_HEADER = "PMTiles"; + + /** + * Parse the header + * @param location The location of the PMtiles (added to the {@link Header}) + * @param inputStream The stream to read + * @return The header + * @throws IOException if the {@link InputStream} had issues + */ + public static Header parse(URI location, InputStream inputStream) throws IOException { + // First do sanity checks + for (var i = 0; i < MAGIC_HEADER.length(); i++) { + if (inputStream.read() != MAGIC_HEADER.charAt(i)) { + throw new IOException("Malformed PMTiles"); + } + } + // We will have read 7 bytes by now, and the next one is the spec version. + if (inputStream.read() != 3) { + throw new IOException("Malformed PMTiles"); + } + // OK. Header is "correct". Now we need to parse the rest of the header. + return new Header(location, nextInt(inputStream), nextInt(inputStream), nextInt(inputStream), nextInt(inputStream), + nextInt(inputStream), nextInt(inputStream), nextInt(inputStream), nextInt(inputStream), + nextInt(inputStream), nextInt(inputStream), nextInt(inputStream), 0x1 == inputStream.read(), + nextCompressionType(inputStream), nextCompressionType(inputStream), nextTileType(inputStream), + inputStream.read(), inputStream.read(), nextDegrees(inputStream), nextDegrees(inputStream), + nextDegrees(inputStream), nextDegrees(inputStream), inputStream.read(), nextDegrees(inputStream), + nextDegrees(inputStream)); + } + + private static double nextDegrees(InputStream inputStream) throws IOException { + return Util.nextInt(inputStream, 4) / 10_000_000d; + } + + private static long nextInt(InputStream inputStream) throws IOException { + return Util.nextInt(inputStream, 8); + } + + private static InternalCompression nextCompressionType(InputStream inputStream) throws IOException { + final var type = inputStream.read(); + return switch (type) { + case 0 -> InternalCompression.UNKNOWN; + case 1 -> InternalCompression.NONE; + case 2 -> InternalCompression.GZIP; + case 3 -> InternalCompression.BROTLI; + case 4 -> InternalCompression.ZSTD; + default -> throw new IllegalStateException("Unexpected value: " + type); + }; + } + + private static TileType nextTileType(InputStream inputStream) throws IOException { + final var type = inputStream.read(); + return switch (type) { + case 0 -> TileType.UNKNOWN; + case 1 -> TileType.MVT; + case 2 -> TileType.PNG; + case 3 -> TileType.JPEG; + case 4 -> TileType.WEBP; + case 5 -> TileType.AVIF; + default -> throw new IllegalStateException("Unexpected value: " + type); + }; + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/Util.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/Util.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/internal/Util.java (revision 36112) @@ -0,0 +1,29 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib.internal; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Utils for reading pmtiles + */ +final class Util { + private Util() {/* Hide constructor */} + + /** + * Read the next int + * @param inputStream The inputstream to read from + * @param width The expected width + * @return The next int + * @throws IOException if there is an issue reading from the stream + */ + static long nextInt(InputStream inputStream, int width) throws IOException { + final var buffer = ByteBuffer.wrap(inputStream.readNBytes(width)).order(ByteOrder.LITTLE_ENDIAN); + if (width == 8) { + return buffer.getLong(); + } + return buffer.getInt(); + } +} Index: applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/package-info.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/package-info.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/main/java/org/openstreetmap/josm/plugins/pmtiles/lib/package-info.java (revision 36112) @@ -0,0 +1,6 @@ +// License: GPL. For details, see LICENSE file. +/** + * This package is not supposed to have any JOSM package usage; this means that other programs should + * be able to use this package as a library without including JOSM. + */ +package org.openstreetmap.josm.plugins.pmtiles.lib; Index: applications/editors/josm/plugins/pmtiles/src/test/java/org/openstreetmap/josm/plugins/pmtiles/lib/PMTilesTest.java =================================================================== --- applications/editors/josm/plugins/pmtiles/src/test/java/org/openstreetmap/josm/plugins/pmtiles/lib/PMTilesTest.java (revision 36112) +++ applications/editors/josm/plugins/pmtiles/src/test/java/org/openstreetmap/josm/plugins/pmtiles/lib/PMTilesTest.java (revision 36112) @@ -0,0 +1,121 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.pmtiles.lib; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.net.URI; + +import org.junit.jupiter.api.Test; + +/** + * Test class for {@link PMTiles} + */ +class PMTilesTest { + private static final URI ODBL_VECTOR_FIRENZE = new File("protomaps(vector)ODbL_firenze.pmtiles").exists() ? + new File("protomaps(vector)ODbL_firenze.pmtiles").toURI() : + URI.create("https://github.com/protomaps/PMTiles/raw/main/spec/v3/protomaps(vector)ODbL_firenze.pmtiles"); + + private static final URI ODBL_RASTER_STAMEN = new File("stamen_toner(raster)CC-BY%2BODbL_z3.pmtiles").exists() ? + new File("stamen_toner(raster)CC-BY%2BODbL_z3.pmtiles").toURI() : + URI.create("https://github.com/protomaps/PMTiles/raw/main/spec/v3/stamen_toner(raster)CC-BY%2BODbL_z3.pmtiles"); + + @Test + void testHeader() { + final var header = assertDoesNotThrow(() -> PMTiles.readHeader(ODBL_VECTOR_FIRENZE)); + assertSame(ODBL_VECTOR_FIRENZE, header.location()); + assertEquals(InternalCompression.GZIP, header.internalCompression()); + assertTrue(header.clustered()); + assertEquals(TileType.MVT, header.tileType()); + assertEquals(108, header.tileEntries()); + assertEquals(108, header.addressedTiles()); + } + + @Test + void testMetadata() { + final var header = assertDoesNotThrow(() -> PMTiles.readHeader(ODBL_VECTOR_FIRENZE)); + final var metadata = assertDoesNotThrow(() -> PMTiles.readMetadata(header)); + assertNotNull(metadata); + assertEquals(4, metadata.size()); + assertEquals("baselayer", metadata.getString("type")); + assertEquals("protomaps 2023-01-18T07:49:39Z", metadata.getString("name")); + } + + @Test + void testRootDirectory() { + final var header = assertDoesNotThrow(() -> PMTiles.readHeader(ODBL_VECTOR_FIRENZE)); + final var root = assertDoesNotThrow(() -> PMTiles.readRootDirectory(header)); + assertNotNull(root); + assertEquals(header.tileEntries(), root.entries().length); + assertEquals(new DirectoryEntry(0, 0, 588, 1), root.entries()[0]); + assertEquals(new DirectoryEntry(317221111, 994639, 17276, 1), root.entries()[47]); + // Any issue with delta encoding will probably show up here + assertEquals(new DirectoryEntry(317301844, 3927477, 31394, 1), root.entries()[107]); + } + + @Test + void testHilbertConversion() { + assertEquals(0, PMTiles.convertToHilbert(0, 0, 0)); + assertEquals(1, PMTiles.convertToHilbert(1, 0, 0)); + assertEquals(2, PMTiles.convertToHilbert(1, 0, 1)); + assertEquals(3, PMTiles.convertToHilbert(1, 1, 1)); + assertEquals(4, PMTiles.convertToHilbert(1, 1, 0)); + assertEquals(5, PMTiles.convertToHilbert(2, 0, 0)); + // This is from https://protomaps.com/blog/pmtiles-v3-hilbert-tile-ids . + assertEquals(36052, PMTiles.convertToHilbert(new TileXYZ(8, 40, 87))); + } + + @Test + void testTileConversion() { + assertEquals(new TileXYZ(0, 0, 0), PMTiles.convertToXYZ(0)); + assertEquals(new TileXYZ(1, 0, 0), PMTiles.convertToXYZ(1)); + assertEquals(new TileXYZ(1, 0, 1), PMTiles.convertToXYZ(2)); + assertEquals(new TileXYZ(1, 1, 1), PMTiles.convertToXYZ(3)); + assertEquals(new TileXYZ(1, 1, 0), PMTiles.convertToXYZ(4)); + assertEquals(new TileXYZ(2, 0, 0), PMTiles.convertToXYZ(5)); + assertEquals(new TileXYZ(8, 40, 87), PMTiles.convertToXYZ(36052)); + } + + @Test + void testTileReading223() throws IOException { + final var header = assertDoesNotThrow(() -> PMTiles.readHeader(ODBL_RASTER_STAMEN)); + final var root = assertDoesNotThrow(() -> PMTiles.readRootDirectory(header)); + final var data = PMTiles.readData(header, PMTiles.convertToHilbert(2, 2, 3), new DirectoryCache(root)); + assertEquals(new DirectoryEntry(14, 169957, 4503, 1), root.entries()[14]); + assertEquals(4503, data.length); + assertEquals((byte) 0x89, data[0]); + assertEquals((byte) 0x50, data[1]); + assertEquals((byte) 0x4e, data[2]); + assertEquals((byte) 0x47, data[3]); + assertEquals((byte) 0x0d, data[4]); + assertEquals((byte) 0x44, data[4498]); + assertEquals((byte) 0xae, data[4499]); + assertEquals((byte) 0x42, data[4500]); + assertEquals((byte) 0x60, data[4501]); + assertEquals((byte) 0x82, data[4502]); + } + + @Test + void testTileReading352() throws IOException { + final var header = assertDoesNotThrow(() -> PMTiles.readHeader(ODBL_RASTER_STAMEN)); + final var root = assertDoesNotThrow(() -> PMTiles.readRootDirectory(header)); + final var data = PMTiles.readData(header, PMTiles.convertToHilbert(3, 5, 2), new DirectoryCache(root)); + assertEquals(new DirectoryEntry(76, 662416, 9344, 1), root.entries()[75]); + assertEquals(9344, data.length); + assertEquals((byte) 0x89, data[0]); + assertEquals((byte) 0x50, data[1]); + assertEquals((byte) 0x4e, data[2]); + assertEquals((byte) 0x47, data[3]); + assertEquals((byte) 0x0d, data[4]); + assertEquals((byte) 0x44, data[9339]); + assertEquals((byte) 0xae, data[9340]); + assertEquals((byte) 0x42, data[9341]); + assertEquals((byte) 0x60, data[9342]); + assertEquals((byte) 0x82, data[9343]); + } +}