Skip to content

demo: make a bitmap of blank tiles instead of uploading them #144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bgilbert merged 3 commits into from
Jun 21, 2026
+104 −11
Merged

demo: make a bitmap of blank tiles instead of uploading them #144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f9c2f5e
demo: get jQuery from the CDN we're already using
bgilbert Jun 11, 2026
4a55303
synctiles: unprettify and gzip JSON
bgilbert Jun 11, 2026
dd914ec
demo: make a bitmap of blank tiles instead of uploading them
bgilbert Jun 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 61 additions & 8 deletions demo/_synctiles.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# _synctiles - Generate and upload Deep Zoom tiles for test slides
#
# Copyright (c) 2010-2015 Carnegie Mellon University
# Copyright (c) 2016-2025 Benjamin Gilbert
# Copyright (c) 2016-2026 Benjamin Gilbert
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of version 2.1 of the GNU Lesser General Public License
Expand All @@ -21,10 +21,12 @@
from __future__ import annotations

from argparse import ArgumentParser, FileType
from array import array
import base64
from collections.abc import Callable, Iterator
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
import gzip
from hashlib import md5, sha256
from io import BytesIO
import json
Expand Down Expand Up @@ -57,7 +59,7 @@
if TYPE_CHECKING:
from mypy_boto3_s3.service_resource import Object

STAMP_VERSION = 'threads' # change to retile without OpenSlide version bump
STAMP_VERSION = 'sparse' # change to retile without OpenSlide version bump
CORS_ORIGINS = ['*']
DOWNLOAD_BASE_URL = 'https://openslide.cs.cmu.edu/download/openslide-testdata/'
DOWNLOAD_INDEX = 'index.json'
Expand Down Expand Up @@ -178,6 +180,12 @@ class ImageInfo(TypedDict):
name: str | None
mpp: float | None
source: DzSource
sparse: dict[str, SparseLevel]


class SparseLevel(TypedDict):
tiles: tuple[int, int]
bitmap: str


class DzSource(TypedDict):
Expand Down Expand Up @@ -262,14 +270,43 @@ def upload_metadata(
self, path: PurePath, item: Any, cache: bool = True
) -> None:
self.object(path).put(
Body=json.dumps(item, indent=1, sort_keys=True).encode(),
Body=gzip.compress(
json.dumps(
item, separators=(',', ':'), sort_keys=True
).encode()
),
CacheControl=(
CACHE_CONTROL_CACHE if cache else CACHE_CONTROL_NOCACHE
),
ContentEncoding='gzip',
ContentType='application/json',
)


class SparseMap:
def __init__(self, generator: Generator):
self._level_tiles = generator.dz.level_tiles
self._bitmaps = [
# ceil division
array('B', [0] * -(level_tiles[0] * level_tiles[1] // -8))
for level_tiles in self._level_tiles
]

def set_bit(self, level: int, address: tuple[int, int]) -> None:
bit = self._level_tiles[level][0] * address[1] + address[0]
self._bitmaps[level][bit >> 3] |= 1 << (bit & 7)

def save(self) -> dict[str, SparseLevel]:
return {
str(level): {
'tiles': self._level_tiles[level],
'bitmap': base64.b64encode(bitmap.tobytes()).decode(),
}
for level, bitmap in enumerate(self._bitmaps)
if any(bitmap)
}


@dataclass
class Tile:
storage: S3Storage
Expand All @@ -279,9 +316,12 @@ class Tile:
key_name: PurePath
cur_md5: str | None

def sync(self) -> PurePath:
def sync(self) -> PurePath | tuple[int, tuple[int, int]]:
"""Generate and possibly upload a tile."""
tile = self.generator.get_tile(self.level, self.address)
if tile.getextrema() == ((255, 255), (255, 255), (255, 255)): # type: ignore[no-untyped-call]
# completely white tile; add to sparse bitmap
return (self.level, self.address)
buf = BytesIO()
tile.save(
buf,
Expand Down Expand Up @@ -352,13 +392,18 @@ def progress() -> None:

# Sync tiles
progress()
sparse_map = SparseMap(generator)
for future in as_completed(
exec.submit(Tile.sync, tile)
for tile in Tile.enumerate(
storage, generator, key_imagepath, key_md5sums
)
):
key_md5sums.pop(future.result(), None)
result = future.result()
if isinstance(result, PurePath):
key_md5sums.pop(result, None)
else:
sparse_map.set_bit(*result)
count += 1
if count % 100 == 0:
progress()
Expand All @@ -385,6 +430,7 @@ def progress() -> None:
'name': associated,
'mpp': mpp,
'source': source,
'sparse': sparse_map.save(),
}


Expand All @@ -403,8 +449,11 @@ def sync_slide(

# Get current metadata
try:
resp = storage.object(metadata_key_name).get()
metadata: SlideMetadata | None = json.load(
storage.object(metadata_key_name).get()['Body']
gzip.open(resp['Body'])
if resp.get('ContentEncoding') == 'gzip'
else resp['Body']
)
except storage.NoSuchKey:
metadata = None
Expand Down Expand Up @@ -629,8 +678,12 @@ def start_retile(

# If the stamp is changing, mark bucket dirty
try:
stream = storage.object(PurePath(METADATA_NAME)).get()['Body']
metadata: BucketMetadata = json.load(stream)
resp = storage.object(PurePath(METADATA_NAME)).get()
metadata: BucketMetadata = json.load(
gzip.open(resp['Body'])
if resp.get('ContentEncoding') == 'gzip'
else resp['Body']
)
old_stamp = metadata['stamp']
except storage.NoSuchKey:
old_stamp = None
Expand Down
46 changes: 43 additions & 3 deletions demo/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
OpenSlide demo site

Copyright (c) 2010-2015 Carnegie Mellon University
Copyright (c) 2016 Benjamin Gilbert
Copyright (c) 2016-2026 Benjamin Gilbert

This program is free software; you can redistribute it and/or modify it
under the terms of version 2.1 of the GNU Lesser General Public License
Expand Down Expand Up @@ -59,7 +59,7 @@
<div id="details" class="scroll-parent"></div>
</div>

<script src="https://code.jquery.com/jquery-4.0.0.min.js" integrity="sha256-OaVG6prZf4v69dPg6PhVattBXkcOWQB62pdZ3ORyrao=" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/jquery@4.0.0/dist/jquery.min.js" integrity="sha256-OaVG6prZf4v69dPg6PhVattBXkcOWQB62pdZ3ORyrao=" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/openseadragon@6.0.2/build/openseadragon/openseadragon.min.js" integrity="sha256-xFw3UC7oKMnWjRwWFCtFNv5UgUx1xnqzFw8aCVkn7UY=" crossorigin="anonymous"></script>
<script src="openseadragon-scalebar.js"></script>
<script>
Expand All @@ -70,6 +70,7 @@
var stamp;
var groups;
var viewer;
var image;
var status_skipped;

function check_status() {
Expand Down Expand Up @@ -114,6 +115,22 @@
});
setInterval(check_status, 300000);

function decode_sparse_bitmaps(image) {
$.each(image.sparse, function(level, sparse) {
if (Uint8Array.fromBase64) {
sparse.bitmap = Uint8Array.fromBase64(sparse.bitmap, {
lastChunkHandling: "strict"
});
} else {
var decoded = atob(sparse.bitmap);
sparse.bitmap = new Uint8Array(decoded.length);
for (var i = 0; i < decoded.length; i++) {
sparse.bitmap[i] = decoded.charCodeAt(i);
}
}
});
}

function populate_images(data, status, xhr) {
stamp = data.stamp;
groups = data.groups;
Expand All @@ -140,7 +157,9 @@
a.data('slide', j);
a.data('image', k);
$('<li/>').appendTo(images).append(a);
decode_sparse_bitmaps(image);
});
decode_sparse_bitmaps(slide.slide);
});
});
var versions = $('<p id="versions">');
Expand Down Expand Up @@ -201,7 +220,6 @@
var slide_id = link.data('slide');
var associated_id = link.data('image');
var slide = groups[group_id].slides[slide_id];
var image;
if (typeof associated_id === 'undefined') {
image = slide.slide;
} else {
Expand Down Expand Up @@ -254,6 +272,28 @@
viewer.source.getTileUrl = function() {
return getTileUrl.apply(this, arguments) + '?v=' + stamp;
};
var tileExists = viewer.source.tileExists;
viewer.source.tileExists = function(level, x, y) {
if (!tileExists.apply(this, arguments)) {
return false;
}
if (!image.sparse) {
return true;
}
var sparse = image.sparse[level];
if (!sparse) {
return true;
}
if (x < 0 || y < 0 ||
x >= sparse.tiles[0] || y >= sparse.tiles[1]) {
return false;
}
var idx = y * sparse.tiles[0] + x;
if (sparse.bitmap[idx >> 3] & (1 << (idx & 7))) {
return false;
}
return true;
};
});
viewer.scalebar({
xOffset: 10,
Expand Down