diff --git a/plugin/cog/cog-commons/src/main/java/it/geosolutions/imageioimpl/plugins/cog/ByteChunksMerger.java b/plugin/cog/cog-commons/src/main/java/it/geosolutions/imageioimpl/plugins/cog/ByteChunksMerger.java new file mode 100644 index 000000000..5a94341be --- /dev/null +++ b/plugin/cog/cog-commons/src/main/java/it/geosolutions/imageioimpl/plugins/cog/ByteChunksMerger.java @@ -0,0 +1,59 @@ +package it.geosolutions.imageioimpl.plugins.cog; + +import it.geosolutions.imageio.utilities.SoftValueHashMap; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ByteChunksMerger { + + private ByteChunksMerger(){} + + public static Map merge(Map data) { + if (data == null || data.isEmpty()) { + return Collections.emptyMap(); + } + + // Sort the entries by their keys (byte offsets) + List> sortedEntries = new ArrayList<>(data.entrySet()); + sortedEntries.sort(Map.Entry.comparingByKey()); + + Map mergedData = new SoftValueHashMap<>(0); + long currentStart = sortedEntries.get(0).getKey(); + byte[] currentBytes = sortedEntries.get(0).getValue(); + + for (int i = 1; i < sortedEntries.size(); i++) { + Map.Entry entry = sortedEntries.get(i); + long nextStart = entry.getKey(); + byte[] nextBytes = entry.getValue(); + + long currentEnd = currentStart + currentBytes.length - 1; + long nextEnd = nextStart + nextBytes.length - 1; + + if (nextEnd <= currentEnd){ + //included -> skip data + } else if (nextStart <= (currentEnd + 1)) { + // intersection or touching + int overlappingElements = (int)(currentEnd - nextStart + 1); + int combinedLen = currentBytes.length + nextBytes.length - overlappingElements; + byte[] combinedBytes = new byte[combinedLen]; + System.arraycopy(currentBytes, 0, combinedBytes, 0, currentBytes.length); + System.arraycopy(nextBytes, overlappingElements, combinedBytes, currentBytes.length, combinedLen - currentBytes.length); + currentBytes = combinedBytes; + } else { + // No overlap, add the current entry to the merged data + mergedData.put(currentStart, currentBytes); + currentStart = nextStart; + currentBytes = nextBytes; + } + + } + + // Add the last entry + mergedData.put(currentStart, currentBytes); + + return mergedData; + } +} diff --git a/plugin/cog/cog-commons/src/test/java/it/geosolutions/imageio/cog/ByteChunksMergerTest.java b/plugin/cog/cog-commons/src/test/java/it/geosolutions/imageio/cog/ByteChunksMergerTest.java new file mode 100644 index 000000000..517ef5f4c --- /dev/null +++ b/plugin/cog/cog-commons/src/test/java/it/geosolutions/imageio/cog/ByteChunksMergerTest.java @@ -0,0 +1,112 @@ +package it.geosolutions.imageio.cog; + +import it.geosolutions.imageio.utilities.SoftValueHashMap; +import it.geosolutions.imageioimpl.plugins.cog.ByteChunksMerger; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; + +public class ByteChunksMergerTest { + + @Test + public void merge_whenNotConnected_expectNoChange(){ + Map orgData = new SoftValueHashMap<>(0); + orgData.put(0L, new byte[] {0,1,2,3}); + orgData.put(10L, new byte[] {10,11,12,13}); + + Map mergedData = ByteChunksMerger.merge(orgData); + + Assert.assertEquals(2, mergedData.size()); + Assert.assertArrayEquals(new byte[]{0,1,2,3}, mergedData.get(0L)); + Assert.assertArrayEquals(new byte[]{10,11,12,13}, mergedData.get(10L)); + } + + @Test + public void merge_whenIntersecting_expectMerged(){ + Map orgData = new SoftValueHashMap<>(0); + orgData.put(0L, new byte[] {0,1,2,3}); + orgData.put(3L, new byte[] {3,4,5}); + + Map mergedData = ByteChunksMerger.merge(orgData); + + Assert.assertEquals(1, mergedData.size()); + Assert.assertEquals(6, mergedData.get(0L).length); + Assert.assertArrayEquals(new byte[]{0, 1, 2, 3, 4, 5}, mergedData.get(0L)); + } + + @Test + public void merge_whenTouching_expectMerged(){ + Map orgData = new SoftValueHashMap<>(0); + orgData.put(0L, new byte[] {0,1,2,3}); + orgData.put(4L, new byte[] {4,5,6,7}); + + Map mergedData = ByteChunksMerger.merge(orgData); + + Assert.assertEquals(1, mergedData.size()); + Assert.assertEquals(8, mergedData.get(0L).length); + Assert.assertArrayEquals(new byte[] {0,1,2,3,4,5,6,7}, mergedData.get(0L)); + } + + @Test + public void merge_whenIncluded_expectMerged(){ + Map orgData = new SoftValueHashMap<>(0); + orgData.put(2L, new byte[]{2,3}); + orgData.put(0L, new byte[]{0,1,2,3}); + + Map mergedData = ByteChunksMerger.merge(orgData); + + Assert.assertEquals(1, mergedData.size()); + Assert.assertArrayEquals(new byte[]{0,1,2,3}, mergedData.get(0L)); + } + + @Test + public void merge_whenCombinationOfEverything_expectCorrectMerged(){ + Map orgData = new SoftValueHashMap<>(0); + orgData.put(0L, new byte[] {0,1,2,3}); + orgData.put(2L, new byte[] {2,3,4,5}); // intersecting + orgData.put(1L, new byte[] {1,2}); // included + orgData.put(10L, new byte[] {10,11,12,13}); // new interval + orgData.put(14L, new byte[] {14,15,16,17}); // touching + orgData.put(100L, new byte[] {100,101}); // not connected + + + Map mergedData = ByteChunksMerger.merge(orgData); + + Assert.assertEquals(3, mergedData.size()); + Assert.assertArrayEquals(new byte[] {0,1,2,3,4,5}, mergedData.get(0L)); + Assert.assertArrayEquals(new byte[] {10,11,12,13,14,15,16,17}, mergedData.get(10L)); + Assert.assertArrayEquals(new byte[] {100,101}, mergedData.get(100L)); + } + + @Test + public void merge_whenGivenNull_expectEmptyMap(){ + Map orgData = null; + + Map mergedData = ByteChunksMerger.merge(orgData); + + Assert.assertEquals(0, mergedData.size()); + } + + @Test + public void merge_whenGivenEmpty_expectEmptyMap(){ + Map orgData = Collections.emptyMap(); + + Map mergedData = ByteChunksMerger.merge(orgData); + + Assert.assertEquals(0, mergedData.size()); + } + + @Test + public void merge_whenSingeEntryMap_expectEmptyMap(){ + Map orgData = new SoftValueHashMap<>(0); + orgData.put(4L, new byte[]{4,5,6,7}); + + Map mergedData = ByteChunksMerger.merge(orgData); + + Assert.assertEquals(1, mergedData.size()); + Assert.assertArrayEquals(new byte[]{4,5,6,7}, mergedData.get(4L)); + } + +} diff --git a/plugin/cog/cog-reader/src/test/java/it/geosolutions/imageio/tiff/CogHTTPReadOnlineTest.java b/plugin/cog/cog-reader/src/test/java/it/geosolutions/imageio/tiff/CogHTTPReadOnlineTest.java index 8a34c3f85..0600390e4 100644 --- a/plugin/cog/cog-reader/src/test/java/it/geosolutions/imageio/tiff/CogHTTPReadOnlineTest.java +++ b/plugin/cog/cog-reader/src/test/java/it/geosolutions/imageio/tiff/CogHTTPReadOnlineTest.java @@ -163,7 +163,7 @@ public void testFetchHeader() throws IOException { @Test public void readCogCaching() throws IOException { - DefaultCogImageInputStream cogStream = new DefaultCogImageInputStream(cogUrl2); + CachingCogImageInputStream cogStream = new CachingCogImageInputStream(cogUrl2); CogImageReader reader = new CogImageReader(new CogImageReaderSpi()); reader.setInput(cogStream); @@ -182,4 +182,71 @@ public void readCogCaching() throws IOException { Assert.assertEquals(height, cogImage.getHeight()); } + + @Test + public void readCogWithCaching_whenReusingCachedTiles_expectNoException() throws IOException { + CachingCogImageInputStream cogStream = new CachingCogImageInputStream(cogUrl2); + CogImageReader reader = new CogImageReader(new CogImageReaderSpi()); + reader.setInput(cogStream); + + CogImageReadParam param = new CogImageReadParam(); + param.setRangeReaderClass(HttpRangeReader.class); + param.setHeaderLength(1024); + int x = 128; + int y = 128; + int width = 256; + int height = 256; + + param.setSourceRegion(new Rectangle(x, y, width, height)); + BufferedImage cogImage1 = reader.read(0, param); + + CachingCogImageInputStream cogStream2 = new CachingCogImageInputStream(cogUrl2); + CogImageReader reader2 = new CogImageReader(new CogImageReaderSpi()); + reader2.setInput(cogStream2); + + CogImageReadParam param2 = new CogImageReadParam(); + param2.setRangeReaderClass(HttpRangeReader.class); + param2.setHeaderLength(1024); + + param2.setSourceRegion(new Rectangle(x, y, width, height)); + BufferedImage cogImage2 = reader2.read(0, param2); + + Assert.assertEquals(cogImage1.getWidth(), cogImage2.getWidth()); + Assert.assertTrue("code above will fail, but should not",true); + } + + @Test + public void readCogWithCaching_whenTilesPartlyCached_expectNoException() throws IOException { + CachingCogImageInputStream cogStream = new CachingCogImageInputStream(cogUrl2); + CogImageReader reader = new CogImageReader(new CogImageReaderSpi()); + reader.setInput(cogStream); + + CogImageReadParam param = new CogImageReadParam(); + param.setRangeReaderClass(HttpRangeReader.class); + param.setHeaderLength(1024); + int x = 128; + int y = 128; + int width = 256; + int height = 256; + + param.setSourceRegion(new Rectangle(x, y, width, height)); + BufferedImage cogImage1 = reader.read(0, param); + + CachingCogImageInputStream cogStream2 = new CachingCogImageInputStream(cogUrl2); + CogImageReader reader2 = new CogImageReader(new CogImageReaderSpi()); + reader2.setInput(cogStream2); + + CogImageReadParam param2 = new CogImageReadParam(); + param2.setRangeReaderClass(HttpRangeReader.class); + param2.setHeaderLength(1024); + int x2 = x + width; + int y2 = y + height; + + param2.setSourceRegion(new Rectangle(x2, y2, width, height)); + BufferedImage cogImage2 = reader2.read(0, param2); + + Assert.assertEquals(cogImage1.getWidth(), cogImage2.getWidth()); + Assert.assertTrue("code above will fail, but should not",true); + } + } diff --git a/plugin/cog/cog-streams/src/main/java/it/geosolutions/imageioimpl/plugins/cog/CacheConfig.java b/plugin/cog/cog-streams/src/main/java/it/geosolutions/imageioimpl/plugins/cog/CacheConfig.java index fb7e4c183..686aea6b7 100644 --- a/plugin/cog/cog-streams/src/main/java/it/geosolutions/imageioimpl/plugins/cog/CacheConfig.java +++ b/plugin/cog/cog-streams/src/main/java/it/geosolutions/imageioimpl/plugins/cog/CacheConfig.java @@ -68,8 +68,8 @@ public class CacheConfig { private static String xmlConfigPath; public CacheConfig() { - useDiskCache = Boolean.getBoolean(getEnvironmentValue(COG_CACHING_USE_DISK, "false")); - useOffHeapCache = Boolean.getBoolean(getEnvironmentValue(COG_CACHING_USE_OFF_HEAP, "false")); + useDiskCache = Boolean.parseBoolean(getEnvironmentValue(COG_CACHING_USE_DISK, "false")); + useOffHeapCache = Boolean.parseBoolean(getEnvironmentValue(COG_CACHING_USE_OFF_HEAP, "false")); diskCacheSize = Integer.parseInt( getEnvironmentValue(COG_CACHING_DISK_CACHE_SIZE, Integer.toString(500 * MEBIBYTE_IN_BYTES))); offHeapSize = Integer.parseInt( diff --git a/plugin/cog/cog-streams/src/main/java/it/geosolutions/imageioimpl/plugins/cog/CachingCogImageInputStream.java b/plugin/cog/cog-streams/src/main/java/it/geosolutions/imageioimpl/plugins/cog/CachingCogImageInputStream.java index 9c2db914d..5face5e51 100644 --- a/plugin/cog/cog-streams/src/main/java/it/geosolutions/imageioimpl/plugins/cog/CachingCogImageInputStream.java +++ b/plugin/cog/cog-streams/src/main/java/it/geosolutions/imageioimpl/plugins/cog/CachingCogImageInputStream.java @@ -17,13 +17,11 @@ package it.geosolutions.imageioimpl.plugins.cog; import it.geosolutions.imageio.core.BasicAuthURI; -import it.geosolutions.imageio.plugins.cog.CogImageReadParam; +import it.geosolutions.imageio.utilities.SoftValueHashMap; -import javax.imageio.stream.ImageInputStreamImpl; -import java.io.IOException; import java.net.URI; import java.net.URL; -import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.logging.Logger; @@ -41,149 +39,109 @@ * @author joshfix * Created on 2019-08-28 */ -public class CachingCogImageInputStream extends ImageInputStreamImpl implements CogImageInputStream { +public class CachingCogImageInputStream extends DefaultCogImageInputStream implements CogImageInputStream { private boolean initialized = false; - protected URI uri; - protected RangeReader rangeReader; - protected CogTileInfo header; + private static final Logger LOGGER = Logger.getLogger(CachingCogImageInputStream.class.getName()); - private final static Logger LOGGER = Logger.getLogger(CachingCogImageInputStream.class.getName()); - - public CachingCogImageInputStream(URI uri) { - this.uri = uri; + public CachingCogImageInputStream(String url) { + super(url); } - public CachingCogImageInputStream(String uri) { - this(URI.create(uri)); + public CachingCogImageInputStream(URL url) { + super(url); } - public CachingCogImageInputStream(URL url) { - this(url.toString()); + public CachingCogImageInputStream(URI uri) { + super(uri); } public CachingCogImageInputStream(BasicAuthURI cogUri) { - this.uri = cogUri.getUri(); + super(cogUri); } public CachingCogImageInputStream(URI uri, RangeReader rangeReader) { - this.uri = uri; - init(rangeReader); - } - - /** - * Directly sets the range reader and reads the header. - * - * @param rangeReader A `RangeReader` implementation to be used. - */ - @Override - public void init(RangeReader rangeReader) { - this.rangeReader = rangeReader; - initializeHeader(); + super(uri, rangeReader); } - /** - * Uses the class specified in `CogImageReadParam` to attempt to instantiate a `RangeReader` implementation, then - * reads the header. - * - * @param param An `ImageReadParam` that contains information about which `RangeReader` implementation to use. - */ - @Override - public void init(CogImageReadParam param) { - Class rangeReaderClass = ((CogImageReadParam) param).getRangeReaderClass(); - if (null != rangeReaderClass) { - try { - rangeReader = rangeReaderClass.getDeclaredConstructor(URI.class, int.class) - .newInstance(uri, param.getHeaderLength()); - } catch (Exception e) { - LOGGER.severe("Unable to instantiate range reader class " + rangeReaderClass.getCanonicalName()); - throw new RuntimeException(e); - } - } else { - throw new RuntimeException("Range reader class not specified in CogImageReadParam."); - } - - if (rangeReader == null) { - throw new RuntimeException("Unable to instantiate range reader class " - + rangeReaderClass.getCanonicalName()); - } - - initializeHeader(param.getHeaderLength()); - } @Override - public boolean isInitialized() { - return initialized; - } - - protected void initializeHeader() { - initializeHeader(CogImageReadParam.DEFAULT_HEADER_LENGTH); - } - protected void initializeHeader(int headerLength) { header = new CogTileInfo(headerLength); + data = new SoftValueHashMap<>(0); // determine if the header has already been cached + byte[] headerBytes; if (CacheManagement.DEFAULT.headerExists(uri.toString())) { - headerLength = CacheManagement.DEFAULT.getHeader(uri.toString()).length; - rangeReader.setHeaderLength(headerLength); + headerBytes = CacheManagement.DEFAULT.getHeader(uri.toString()); + rangeReader.setHeaderLength(headerBytes.length); } else { - CacheManagement.DEFAULT.cacheHeader(uri.toString(), rangeReader.fetchHeader()); + headerBytes = rangeReader.readHeader(); + CacheManagement.DEFAULT.cacheHeader(uri.toString(), headerBytes); } + data.put(0L, headerBytes); initialized = true; } @Override - public CogTileInfo getHeader() { - return header; + public boolean isInitialized() { + return initialized; } /** * TIFFImageReader will read and decode the requested region of the GeoTIFF tile by tile. Because of this, we will * not arbitrarily store fixed-length byte chunks in cache, but instead create a cache entry for all the bytes for * each tile. - *

- * The first step is to loop through the tile ranges from CogTileInfo and determine which tiles are already cached. - * Tile ranges that are not in cache are submitted to RangeBuilder to build contiguous ranges to be read via HTTP. - *

- * Once the contiguous ranges have been read, we obtain the full image-length byte array from the RangeReader. Then - * loop through each of the requested tile ranges from CogTileInfo and cache the bytes. - *

- * There are likely lots of optimizations to be made in here. */ @Override public void readRanges(CogTileInfo cogTileInfo) { - // instantiate the range builder + + Map missingTiles = loadDataFromCache(cogTileInfo); + + // fetch the missing tiles ContiguousRangeComposer contiguousRangeComposer = - new ContiguousRangeComposer(0, cogTileInfo.getHeaderLength() - 1); + new ContiguousRangeComposer(0L, cogTileInfo.getHeaderLength() - 1L); + missingTiles.forEach((tileIndex, tileRange) -> { + if (tileIndex == HEADER_TILE_INDEX) { + return; + } + contiguousRangeComposer.addTileRange(tileRange.getStart(), tileRange.getEnd()); + }); + rangeReader.setHeaderLength(cogTileInfo.getHeaderLength()); + Set ranges = contiguousRangeComposer.getRanges(); + LOGGER.fine("Submitting " + ranges.size() + " range request(s)"); + Map fetchedTiles = rangeReader.read(ranges); + this.data.putAll(fetchedTiles); + + //create contiguous data chunks (required for super.read(...)) + this.data = ByteChunksMerger.merge(this.data); + + //populate cache + updateTileCache(missingTiles); - // determine which requested tiles are not in cache and build the required ranges that need to be read (if any) + } + + private Map loadDataFromCache(CogTileInfo cogTileInfo) { + Map missingTiles = new HashMap<>(); cogTileInfo.getTileRanges().forEach((tileIndex, tileRange) -> { if (tileIndex == HEADER_TILE_INDEX) { return; } - TileCacheEntryKey key = new TileCacheEntryKey(uri.toString(), tileIndex); - if (!CacheManagement.DEFAULT.keyExists(key)) { - contiguousRangeComposer.addTileRange(tileRange.getStart(), tileRange.getEnd()); + if (CacheManagement.DEFAULT.keyExists(key)) { + data.put(tileRange.getStart(), CacheManagement.DEFAULT.getTile(key)); + } else { + missingTiles.put(tileIndex, tileRange); } }); - rangeReader.setHeaderLength(cogTileInfo.getHeaderLength()); - // get the ranges for the tiles that are not already cached. if there are none, simply return - Set ranges = contiguousRangeComposer.getRanges(); - if (ranges.size() == 0) { - return; - } + return missingTiles; + } - // read all they byte ranges for tiles that are not in cache - LOGGER.fine("Submitting " + ranges.size() + " range request(s)"); - // Update the headerLength - Map data = rangeReader.read(ranges); - // cache the bytes for each tile - cogTileInfo.getTileRanges().forEach((tileIndex, tileRange) -> { + private void updateTileCache(Map missingTiles) { + missingTiles.forEach((tileIndex, tileRange) -> { if (tileIndex == HEADER_TILE_INDEX) { return; } @@ -191,49 +149,17 @@ public void readRanges(CogTileInfo cogTileInfo) { for (Map.Entry entry : data.entrySet()) { long contiguousRangeOffset = entry.getKey(); - int contiguousRangeLength = (int) contiguousRangeOffset + entry.getValue().length; + byte[] contiguousBytes = entry.getValue(); + int contiguousRangeLength = (int) contiguousRangeOffset + contiguousBytes.length; if (tileRange.getStart() >= contiguousRangeOffset && tileRange.getEnd() < contiguousRangeLength) { - byte[] contiguousBytes = entry.getValue(); - long relativeOffset = tileRange.getStart() - contiguousRangeOffset; - byte[] tileBytes = Arrays - .copyOfRange(contiguousBytes, (int) relativeOffset, (int) tileRange.getEnd()); + int relativeOffset = (int) (tileRange.getStart() - contiguousRangeOffset); + int tileByteLen = (int) tileRange.getByteLength(); + byte[] tileBytes = new byte[tileByteLen]; + System.arraycopy(contiguousBytes, relativeOffset, tileBytes, 0, tileByteLen); CacheManagement.DEFAULT.cacheTile(key, tileBytes); } } }); } - @Override - public int read() throws IOException { - byte[] b = new byte[1]; - read(b, 0, 1); - return b[0]; - } - - @Override - public int read(byte[] b, int off, int len) { - // based on the stream position, determine which tile we are in and fetch the corresponding TileRange - // TODO: CachingCogImageInputStream never worked very well. - // We need to update this section too, when fixing the related ticket - TileRange tileRange = header.getTileRange(streamPos); - - // get the bytes from cache for the tile. need to determine if we're reading from the header or a tile. - byte[] bytes; - switch (tileRange.getIndex()) { - case HEADER_TILE_INDEX: - bytes = CacheManagement.DEFAULT.getHeader(uri.toString()); - break; - default: - TileCacheEntryKey key = new TileCacheEntryKey(uri.toString(), tileRange.getIndex()); - bytes = CacheManagement.DEFAULT.getTile(key); - } - - // translate the overall stream position to the stream position of the fetched tile - int relativeStreamPos = (int) (streamPos - tileRange.getStart()); - - // copy the bytes from the fetched tile into the destination byte array - System.arraycopy(bytes, relativeStreamPos, b, off, len); - streamPos += len; - return len; - } }