From 37b239b83a06598afb03d086b86a46bf2597f840 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 1 Oct 2026 13:53:25 -0700 Subject: [PATCH] feat: Add big endian fixtures with predictor 2 and predictor 3 GDAL's COG driver can't write big endian files, so these are rewritten from a little endian COG through the GTiff driver with COPY_SRC_OVERVIEWS, which keeps the COG layout. - uint16_1band_lzw_block128_predictor2_big_endian: readers must swap samples to native order before undoing horizontal differencing. - float32_1band_deflate_block64_predictor3_big_endian: predictor 3 byte planes are most significant byte first in every file, so readers must not swap these samples. A test checks that both files are MM, carry their predictor on every IFD, and match their little endian source at every level. --- ...d_deflate_block64_predictor3_big_endian.py | 36 +++++++++ ..._deflate_block64_predictor3_big_endian.tif | Bin 0 -> 4517 bytes ...late_block64_predictor3_big_endian_info.md | 47 +++++++++++ ...band_lzw_block128_predictor2_big_endian.py | 23 ++++++ ...and_lzw_block128_predictor2_big_endian.tif | Bin 0 -> 2785 bytes ...lzw_block128_predictor2_big_endian_info.md | 47 +++++++++++ rasterio_generated/write_utils.py | 26 ++++++ tests/test_big_endian.py | 75 ++++++++++++++++++ 8 files changed, 254 insertions(+) create mode 100644 rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian.py create mode 100644 rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian.tif create mode 100644 rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian_info.md create mode 100644 rasterio_generated/fixtures/uint16_1band_lzw_block128_predictor2_big_endian.py create mode 100644 rasterio_generated/fixtures/uint16_1band_lzw_block128_predictor2_big_endian.tif create mode 100644 rasterio_generated/fixtures/uint16_1band_lzw_block128_predictor2_big_endian_info.md create mode 100644 tests/test_big_endian.py diff --git a/rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian.py b/rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian.py new file mode 100644 index 0000000..2d5a935 --- /dev/null +++ b/rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian.py @@ -0,0 +1,36 @@ +"""Generate a big endian (MM) float32 GeoTIFF with the floating point predictor.""" + +import tempfile +from pathlib import Path + +import numpy as np + +from rasterio_generated.write_utils import rewrite_big_endian, write_cog + +HERE = Path(__file__).parent + + +def generate_source(output_path: Path) -> None: + """Generate the little endian COG this fixture is rewritten from.""" + data = np.linspace(0.0, 1.0, 128 * 128, dtype=np.float32).reshape(128, 128) + + write_cog( + output_path, + data, + blocksize=64, + compress="DEFLATE", + predictor=3, + nodata_type=None, + ) + + +def generate(output_path: Path) -> None: + """Generate a 128x128 tiled float32 big endian GeoTIFF with DEFLATE and predictor 3. + + Predictor 3 splits each row into byte planes, most significant byte first, + whatever the file's byte order, so readers must not swap these samples. + """ + with tempfile.TemporaryDirectory() as tmp: + src_path = Path(tmp) / "source.tif" + generate_source(src_path) + rewrite_big_endian(src_path, output_path) diff --git a/rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian.tif b/rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian.tif new file mode 100644 index 0000000000000000000000000000000000000000..4cae054a6ca729753976daf0510a6e6f11f1369d GIT binary patch literal 4517 zcmbVNYgiN4)(#+oARt<#s1Uh%sTaI!i#S|`2pG#nykRt_Vq3HUMGz@&KoOM+1xh2{ z0yT=BJ~dF<8l^J$Q4U%TO^UUIf{j!tNr|N5lr%HU>~94<{rdMi^E^AVve&!bwf0)? zPGVzuukm=i=E(4n7*TveTw+*4VjP&U5eXsTAqgR(c)_-ape0L|EM2i=wm3Uu&q0Tn zkgdYRgdoBC@OV*Z#Cl;|gvf$)2#pbjMT>-S;Sq5`al#}~Y{&*-+{@jVh!D%Rg-jF@ zFG>__Ojz;Kl@J#qhyh}M!vd66A|1b8KGjB3rEG(Cf7mV8z!Tzd9 zF^;iR_G?ky_&#K{o>NrMgEtSVv|a^k9dZ0fU*wHfa{luM-+%X5L4SBWJ4?Y}Sx>Nh zgFQ$D?e$Sw(s930Y$?zF_6cb6v^XFUI?W> z0gPpOk6#&=Ih$I0GI^~rCiT*xiknQAnk>&DhZR%81b z^qGd=K%EQPK>LdE9@tYI!sPsyg;i(+o&}dWjD|3!hgd4*Ei^Ru>ABYhN-ZOF} zFbKCD>DU=bN4Ij`5bIKIL@UMU+s@Qco?ha{L_&nA!i0;(SP2o8a$KUpKY{n698b~{ z18G&fM#oIyq{?3;cS%)Q+tPdV+`A=XLvk$D5Kfu>B44&pjC1OIu~Gb;X#Oag5I8az zg@O&L0#7q4=l&tabHTYxFPYAed7fsnf^)7%&W3x0kOrTRrVKC88(kS`bIF)YfMgnM z1HNdwLW~_WWkeVqFUGF{$YwCWV;a*1sO55E(Wv5>r@1x?tv8r}^T90g@J>U3b>Il; z&AwhwzGAqCPgCwE**&`=IcYy5D%6mUz`BwCwDl-oQ-(gGx=yhmaR<&;Y?`v;(zeiP{~Mz0#~ zX@&IMZ#Z<6I9Z3gP3435DX%4~0P(&!*KKE66JA`EA0=UG}`dq%y+#p|A^`=9_yh5LkLR~kc zXY?e}-(EUu%6_)?W{yymyL0Tj!PZ;Jl6vEnWoEO4GqJLmY#5LLB#WSpS@jtf5Mz+*W(A$Zw(yun5)8-t%ZLC{PF&84`&?EpGn`% z=E{+m-ssIfl%ut5ObX4F_m|I>5kKN{6yd|d_*QhHDk`a%Jt)AgohaELDsQ4ac4kf7 zaat)8{Y0LlBy-qBVzfXnilmFa;r7dstsZ%g0%amx8Vx#21~;yfjr!UU9Y z9&-G;Uicw*{wh-vbo$`^%UrJ6Yls}ueaQyu1Ew+jH;(Mb;q=UJxV&{nNZzD*$220FOunJC*|{6~#;fg?%foLP){kg$LXn98{1 zdz#w?@asa@XZ8p9vS-9N9ke}PFPX*^K{5VvGTK1cf^QC+)5`4zpR1mDZB%9xIO1N; zEf-^Zpvt2}u1o;b_gsLFYY2zTew8m<+RCLu<{h1Qy@3z#y%n57j#uB@^)+Q#kKivG z{x*IhqCQHW)%Gms7yD!76@D|~Uh$i;p}?=;>#(|cb!gY7UE8WR?ZVB~&BF)M>_2}y z%|7lOrT4v0?s`Z0Pub?BWr&6=4$iLN9##DnyDe*8g&Up zt0~oawneSI|7XYbZajhRIj8CG;G}A;H5<`@ov5xbe4!h=(hVPIzqVp+b$FIR(9g|n zyXu-tZ&>yN`KsQ@bVwI)meu;)>Eou+%)hzHD2Y-LitZb*D>#j6V$Y}|;z&#GBzmxe z)0X|5hOOBJDagNb=CEUwNE+S2J%vqKNBuiXm{i|8ed}!4>;@#1ukNj_A-fEc52zD4 z9&~XE=73ljjOE?h89 zAJDdp0saYYeFOeL_buYn+dDYNPZr=Ql>5KAVlX^)#NJWS^klPL4XHAaKDS3iMO;q` zUip0`$)}@Ja2|xY(XuLbb$AlzrIRgVB&TnTNvrY2Zj_WUU_8NPH{eSE@}k{4IJ-|4 zAa_b}mUW3p;^Nh0%4q4y8)Fxno1SQcRx%H9S0#D*M91O=T(A39>&!Z(;Ij}qnRcmS zRS>aC2LpKmW$o4xsiWkSWHU@j(M@>Y9^vP4+f(oYD9aYwfwpE0ND8>x6rB2fCE`=U zbL;~M+o?kk=ALXGszKiwWUk5PgKGTHfM?$xF&^jkG@#8H1B(3T?FQ_3dt`4ui%r-` zhaI=Jem-VdiWeg95x%Xmh+K7IMfv*9s2Kv^cl^$Lp-g}BuQM}Ozt=ktzRAm8x|X9s zJjQ3Rf@%VwCbNvM!N3izD|h}+6E+f3|6B=PboQRIJyR3g?Pta zoX<5v*#0!^Nb>=m)y%r+&_7GswYDaIHI4(c-7p!7>H*E~MJKqakj+Yo?b$037M3QO z%svFeHw`<}=O3n<+a7aHRMw|YMug4m0v&&I+knvCaJrh{$acbU*s+EXcuJ9TFPX=^ z1w^;f@GLqWs;#hY+^d#BT7p09SWUzN$f}pf=Q03Pn1)}ams>zKjge|H9(+rC2gC4t z;PX$zJlX_x2NW`lniP3T3IDoAJJt?Lk+$S|QI-lsR;H|Kd<~kqK}kBXZQzTyY22V&*6q@ zqKV8fL$5t_rAF0`qqhks@O8wn6=T<$tSqbPg>1Y3BU`Gce9P2mv0&jT|9Zbw?(g-! zkgogIzcxSqrt>U*jyRQ!9anDq(82C^o@O7u6EKu@IA|!n+h}6A1J;dkO5ujO=u;Q( zKbqh4-6X@MyX+dO7CJ~2F>mUG@ofWWlBq>4y8$|z0g5HUJf(P1uPF?HU~r()Sja7e z#=E3xoyNN*d)zdQz6k?I`AnuxdmX~uhPT(Cn}E(zN=|Thpj$*=lPhfk5I(^i(Gk}G zWH+2zgX;i9C>1BT%K%DD!+iQ63@7z57eEcdwE*e*Q-ct;x|t2o1#AKh7chLa2;#NY zCT%qt1@d$nUQkVL0mEjP2t%hpr|!MzIQQpR_kb~D^opstps1i959qGUg5$Et;9E42Z3Zwg|bIou-pr zE@_ud)l>#!Yv@8%8n&mUpaN9^Q=}$CK`N_yi9Bv49Ors|@={6yHOm}LrfO6SRBM%_ zBijQ?9&BMfb=n%BnyzsR!HdULjvQ$dPz4k)=hdVTsFwC>^Eexz(&s08Q~WqkvV*Cs z+H&*pG9~FSu4HKo>!u?<11eXITQJTBs>(Dpi4K6aDGHckHA)^=xhRk80jjF}Xrlb(J*ETa-ErRk$*I{7@x&oj%N@5cGC4kQNaTBPU npqIUNnYYy1PXScfQBe~6z!PEqshazJ^P4X5KHN@PS=Rpo6*%Gj literal 0 HcmV?d00001 diff --git a/rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian_info.md b/rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian_info.md new file mode 100644 index 0000000..d025b63 --- /dev/null +++ b/rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian_info.md @@ -0,0 +1,47 @@ +``` +Driver: GTiff +File: rasterio_generated/fixtures/float32_1band_deflate_block64_predictor3_big_endian.tif +COG: True +Compression: DEFLATE +ColorSpace: None + +Profile + Width: 128 + Height: 128 + Bands: 1 + Tiled: True + Dtype: float32 + NoData: None + Alpha Band: False + Internal Mask: False + Interleave: BAND + ColorMap: False + ColorInterp: ('gray',) + Scales: (1.0,) + Offsets: (0.0,) + +Geo + Crs: EPSG:4326 + Origin: (0.0, 0.0) + Resolution: (0.01, -0.01) + BoundingBox: (0.0, -1.28, 1.28, 0.0) + MinZoom: 7 + MaxZoom: 7 + +Image Metadata + AREA_OR_POINT: Area + +Image Structure + LAYOUT: COG + COMPRESSION: DEFLATE + INTERLEAVE: BAND + PREDICTOR: 3 + +Band 1 + ColorInterp: gray + +IFD + Id Size BlockSize Decimation + 0 128x128 64x64 0 + 1 64x64 64x64 2 +``` diff --git a/rasterio_generated/fixtures/uint16_1band_lzw_block128_predictor2_big_endian.py b/rasterio_generated/fixtures/uint16_1band_lzw_block128_predictor2_big_endian.py new file mode 100644 index 0000000..891f78b --- /dev/null +++ b/rasterio_generated/fixtures/uint16_1band_lzw_block128_predictor2_big_endian.py @@ -0,0 +1,23 @@ +"""Generate a big endian (MM) copy of uint16_1band_lzw_block128_predictor2.""" + +import tempfile +from pathlib import Path + +from rasterio_generated.fixtures.uint16_1band_lzw_block128_predictor2 import ( + generate as generate_source, +) +from rasterio_generated.write_utils import rewrite_big_endian + +HERE = Path(__file__).parent + + +def generate(output_path: Path) -> None: + """Generate a 128x128 tiled uint16 big endian GeoTIFF with LZW and predictor 2. + + Readers must swap samples to native order before undoing horizontal + differencing, which adds neighbouring samples as integers. + """ + with tempfile.TemporaryDirectory() as tmp: + src_path = Path(tmp) / "source.tif" + generate_source(src_path) + rewrite_big_endian(src_path, output_path) diff --git a/rasterio_generated/fixtures/uint16_1band_lzw_block128_predictor2_big_endian.tif b/rasterio_generated/fixtures/uint16_1band_lzw_block128_predictor2_big_endian.tif new file mode 100644 index 0000000000000000000000000000000000000000..031d55708afb6e42bd281e0cab057bea1697245f GIT binary patch literal 2785 zcmeH{X;f257RT#70z4puup;5`l!|0fI}=K@k;EL|kbDLBojR z0`1m-P8(&B21I0W1r$_7R61gNkP&Qfv;~_M1O<&#LAsA~PWOlDbLQK8xK+Qpb?erx z`k$(xATSjGP%GdOfpQrs4ds%dQiwr(lHd^}A(sgw`D`YW>E+F2EZdT}e1mZy5g`sG z*}{1|nViF)CzkT%x=3SApqM*fE|&86QnpkaE)OC^V(GXykWc8`x@2-fCJz+`liuS~ zq?8Z_!nr^~M#{bAoCuOHlS}y$K0)$%#`A;4;lXlYFjpKTAxI%7kT2)+grra$%nlYC zGXSte0MLUcK!5@^$Y+d>GadT%##tY-!8pT(i1laeFwQ9Svp-|bIOEXI{fv2l0{4Zc zM?v_%vlSv6kAeA>G(d$hus); z9EqOcXqwJ_8e*poo^HJh1bA)kUgke(UqG~lp9e*M;sH_g!5XqM!WbxeEZ!xy#rl2= z@GMrv(E7Q@`8Xq@2i;4YyjjawHQgmwnBSyeY>nf5A94l|){zZKr9zOp`kRqAht?wc z3wqlwx_Q{Hs%)PT5{1p`+Hh&n3ebM}=YcIiSz0yvrUEsq>sogx3gcR4G%P^<-e+?1 z5ToU@9xe0}`{dkd;T-)yOU<@!jjbqWU<+ku9xO&jK4jLrAk?@o+anY)8!vU-;x5$E z`q7C`*ZI}8h;2m(ZD7ly{^9w#IY2ei@l~y|>R~`gQb`5g76Je$=LZ{~k|UF0&{M z?CkILG23NsHK)+P%X?m-K?oU~37m5lTQ?%=7~2A(INsiasMzG-T#Lm|x^WO&?-GCo zpbobzJhL5zE&-}8&r;+BLucE@v|`|N${%aRm~6he zdo9~>zo1+4+%+@EIa}BxJt8si@%*NFg(WN7N?-5T9e3H$Za@8*dG}%xS^H`0d1^^v zNUbB9B|fEDii_By?M`!vja!WV=y8dZwXO5Geqm=U0>)mYnw28Z$gHs|rTap%--0ty~F_vr) zU96r<())Idx;rk($#+^M4iyOPdiBOUcYH6 zEq;Ea`SXDbAC|QJ==SJkZ2jW9v}pIvSMM(6<dEt~ zsds$GCnz-G|C8+V4x_dRY+`XF#`* zUf9_!w(-Dw!>#6dZC|O@yPp&F59R%eq#Wh1ns|nw)jm<7(AoAe3E+1^{bLErTz;*- zp72Qm1(n8iKhU(?qK5R;vp*EL{M;0E3k}T*I(@E2GsHlThzn#*uFZz5?^;(T)&QDP zR3MuwsdaP74=-rNvqu_|&V7MCI5RYRu)t(ddE}b%bw0Zv#-Hbbwv3%^KT-7BONK9& zIlb6k8(C+2rqZSnyV`X&6a-u9?JG=O&!V29T4y#qi-{`ETg~>;e>Up^>&Q%hwI-nM zn5VW9r~dG=G6Y|HPF9tnO!D8`J;$-~QU6lV_rxU4tTTR#1m9tMxCu+x++M5l&o99S zYd&OF#jmyK+xP~V@k@rcW&iLctvqbclotcb(Y(8vcGLQYK3qATvw30vNEtXc3}%3f zS}k7YK)om~)=zpRCMu*Sfw=WCdW$2PE zbdw&9F0pamdwYfSMNV6+p}FwcKy3G`WV(4^%pj}JX65AEkU~-vohIO%(!`$ki#nJ^ z7L$b>(asf@Baa!a8ZvSFI?S!)QCwIoe(c7Y>d2?)>Y&EHgjA!x;f$Wg;*D22<5Dv^ zo|r~kMKRxANY6J(y&04 z#ts)a?5`3)DRw+uPWXz%lrq$CZfuG#i)dR)!8y0^J!4{K5K^J~X75BF?Mrrnj- zR8L46wLfrp+l_B0yuGO1=UW=!{btMSlj8W>G@rMlS3b(y@HOwRG-GSD__XwRFt^Ox zh*Fwz-5f0;uG?bMx|$~$aLRI+=M5#7Jjx9vny-os*A4nw@Iq!+IPvb=f9J&aN=SEV uI7`OZH8x3-?HffN$qs_TA9pwjW?7}U None: + """Rewrite a COG as a big endian (MM) TIFF, keeping its layout. + + GDAL's COG driver has no ENDIANNESS option, so this copies through the GTiff + driver with COPY_SRC_OVERVIEWS, which keeps the COG layout: header and IFDs + first, then overview tiles, then full resolution tiles. Tiling, compression + and predictor are taken from the source. + """ + with rasterio.open(src_path) as src: + block_height, block_width = src.block_shapes[0] + image_structure = src.tags(ns="IMAGE_STRUCTURE") + creation_options = { + "driver": "GTiff", + "tiled": True, + "blockxsize": block_width, + "blockysize": block_height, + "compress": image_structure["COMPRESSION"], + "interleave": image_structure["INTERLEAVE"], + "endianness": "BIG", + } + if "PREDICTOR" in image_structure: + creation_options["predictor"] = image_structure["PREDICTOR"] + + copy(src, path, copy_src_overviews=True, **creation_options) diff --git a/tests/test_big_endian.py b/tests/test_big_endian.py new file mode 100644 index 0000000..1493c0d --- /dev/null +++ b/tests/test_big_endian.py @@ -0,0 +1,75 @@ +"""Check the big endian fixtures. + +GDAL's COG driver can't write big endian files, so these fixtures are rewritten +from a little endian COG through the GTiff driver. Check that they really are +big endian, carry the predictor they are named for on every IFD, and hold the +same pixels as their little endian source at every level. +""" + +from collections.abc import Callable +from pathlib import Path + +import numpy as np +import pytest +import rasterio +import tifffile + +from rasterio_generated.fixtures import ( + float32_1band_deflate_block64_predictor3_big_endian as float32_predictor3, +) +from rasterio_generated.fixtures import ( + uint16_1band_lzw_block128_predictor2 as uint16_predictor2, +) + +FIXTURES_DIR = Path(__file__).parent.parent / "rasterio_generated" / "fixtures" + +CASES = [ + ( + "uint16_1band_lzw_block128_predictor2_big_endian", + 2, + uint16_predictor2.generate, + ), + ( + "float32_1band_deflate_block64_predictor3_big_endian", + 3, + float32_predictor3.generate_source, + ), +] + + +def read_levels(path: Path) -> list[np.ndarray]: + """Read the full resolution image and every overview.""" + with rasterio.open(path) as src: + levels = [src.read()] + overview_count = len(src.overviews(1)) + for level in range(overview_count): + with rasterio.open(path, overview_level=level) as src: + levels.append(src.read()) + return levels + + +@pytest.mark.parametrize(("name", "predictor", "generate_source"), CASES) +def test_big_endian_fixture( + name: str, + predictor: int, + generate_source: Callable[[Path], None], + tmp_path: Path, +): + path = FIXTURES_DIR / f"{name}.tif" + assert path.read_bytes()[:2] == b"MM" + + with tifffile.TiffFile(path) as tif: + assert len(tif.pages) > 1 + assert [int(page.predictor) for page in tif.pages] == [predictor] * len( + tif.pages + ) + + source_path = tmp_path / "source.tif" + generate_source(source_path) + assert source_path.read_bytes()[:2] == b"II" + + actual = read_levels(path) + expected = read_levels(source_path) + assert len(actual) == len(expected) + for actual_level, expected_level in zip(actual, expected): + np.testing.assert_array_equal(actual_level, expected_level)