Improve docs for reading from a file (#95)

* Improve docs for reading from a file

* add deps badge

* update comment

* Use read instead of mmap in benchmark

Author: nhz2

README.md

@@ -4,6 +4,7 @@
 [![CI](https://github.com/JuliaIO/ZipArchives.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/JuliaIO/ZipArchives.jl/actions/workflows/CI.yml)
 [![codecov](https://codecov.io/gh/JuliaIO/ZipArchives.jl/branch/main/graph/badge.svg?token=K3J0T9BZ42)](https://codecov.io/gh/JuliaIO/ZipArchives.jl)
 [![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl)
+[![deps](https://juliahub.com/docs/General/ZipArchives/stable/deps.svg)](https://juliahub.com/ui/Packages/General/ZipArchives?t=2)
 
 Read and write Zip archives in Julia.
 
@@ -93,20 +94,10 @@ end
 ```
 
 ### Streaming one entry in a large archive file
-If your archive is in a file, `mmap` can be used to treat the file as a `Vector{UInt8}`.
+If your archive is in a larger than memory file, `Mmap.jl`
+or a custom struct can be used to treat the file as a `AbstractVector{UInt8}`.
 
-An entry in the archive can be opened as an `IO` stream using `zip_openentry`.
-
-```julia
-using ZipArchives: ZipReader, zip_openentry
-using Downloads: download
-using Mmap: mmap
-zip_file_path = download("https://github.com/JuliaIO/ZipArchives.jl/archive/refs/heads/main.zip");
-archive = ZipReader(mmap(open(zip_file_path)))
-readme_n_lines = zip_openentry(archive, "ZipArchives.jl-main/README.md") do io
-    countlines(io)
-end
-```
+See [test/test_file-array.jl](https://github.com/JuliaIO/ZipArchives.jl/blob/main/test/test_file-array.jl) for an example.
 
 ### Supported Compression Methods
 
@@ -120,7 +111,7 @@ end
 
 1. Cannot directly extract all files in an archive and write those files to disk.
 1. Ignores time stamps.
-1. Cannot write an archive fully in streaming mode. See https://github.com/madler/zipflow if you need this functionality.
+1. Cannot write an archive fully in streaming mode. See https://github.com/madler/zipflow or https://github.com/reallyasi9/ZipStreams.jl if you need this functionality.
 1. Encryption and decryption are not supported.
 1. Multi disk archives not supported.
 1. Cannot recover data from a corrupted archive. Especially if the end of the archive is corrupted.
@@ -149,9 +140,16 @@ Currently, ZipArchives has the following benefits over ZipFile:
 ZipArchives currently has the following limitations compared to ZipFile:
 1. No way to specify the modification time, times are set to 1980-01-01 00:00:00 DOS date time.
 2. No `flush` function for `ZipWriter`. `close` and `zip_append_archive` can be used instead.
-3. No way to read an archive from an `IOStream`, `mmap` can be used instead.
+3. No way to read an archive from an `IOStream`, instead `Mmap.jl` or a custom struct can be used to treat the file as a `AbstractVector{UInt8}`. Example in [test/test_file-array.jl](https://github.com/JuliaIO/ZipArchives.jl/blob/main/test/test_file-array.jl).
+
+### [ZipStreams](https://github.com/reallyasi9/ZipStreams.jl)
+
+Unlike ZipArchives, ZipStreams is able to read from non-seekable streams, but may fail to correctly 
+read some pathological archives.
 
+### [LibZip](https://github.com/bhftbootcamp/LibZip.jl)
 
+LibZip is a wrapper of the [libzip C library](https://github.com/nih-at/libzip) which supports encryption and decryption.
 
 
 ## Is there an unzip function for a whole archive?

benchmark/benchmarks.jl

@@ -29,3 +29,68 @@ ZipWriter(sink) do w
 end
 data = take!(sink)
 rbench["empty ZipReader"] = @benchmarkable ZipReader($(data))
+
+# Reading from a file
+rfbench = rbench["file"] = BenchmarkGroup()
+
+fname = tempname()
+ZipWriter(fname) do w
+    zip_newfile(w, "test/compressed.txt"; compress=true)
+    write(w, "Compressed \n Data \n"^1000)
+    zip_newfile(w, "test/uncompressed.txt")
+    write(w, "Uncompressed Data \n"^1000)
+end
+
+# Based on DiskArrays code suggested by Fabian Gans in 
+# https://discourse.julialang.org/t/struggling-to-use-mmap-with-ziparchives/129839/19
+struct FileArray <: AbstractVector{UInt8}
+    filename::String
+    offset::Int64
+    len::Int64
+end
+function FileArray(filename::String, offset::Int64=Int64(0))
+    len = filesize(filename)
+    len ≥ 0 || error("filesize of $(repr(filename)) is negative")
+    offset ≥ 0 || error("offset $(offset) is negative")
+    offset ≤ len || error("offset $(offset) is larger than the filesize $(len)")
+    FileArray(filename, offset, len-offset)
+end
+Base.size(s::FileArray) = (s.len,)
+function Base.getindex(s::FileArray, i::Int)::UInt8
+    copyto!(zeros(UInt8, 1), Int64(1), s, Int64(i), Int64(1))[1]
+end
+function Base.view(s::FileArray, inds::UnitRange{Int64})::FileArray
+    checkbounds(s, inds)
+    FileArray(s.filename, s.offset + first(inds) - Int64(1), length(inds))
+end
+dest_types = if VERSION ≥ v"1.11"
+    (Vector{UInt8}, Memory{UInt8},)
+else
+    (Vector{UInt8},)
+end
+for dest_type in dest_types
+    @eval begin
+        function Base.copyto!(dest::$dest_type, dstart::Int64, src::FileArray, sstart::Int64, n::Int64)
+            iszero(n) && return dest
+            n ≥ 0 || throw(ArgumentError("tried to copy n=$(n) elements, but n should be non-negative"))
+            checkbounds(dest, dstart)
+            checkbounds(src, sstart)
+            checkbounds(dest, dstart + n - Int64(1))
+            checkbounds(src, sstart + n - Int64(1))
+            open(src.filename) do io
+                seek(io, src.offset + sstart - Int64(1))
+                nb = readbytes!(io, view(dest, range(dstart; length=n)))
+                nb == n || error("short read")
+            end
+            return dest
+        end
+    end
+end
+
+r = ZipReader(FileArray(fname))
+rfbench["ZipReader FileArray"] = @benchmarkable ZipReader(FileArray($(fname)))
+rfbench["zip_test FileArray"] = @benchmarkable zip_test($(r))
+
+r = ZipReader(read(fname))
+rfbench["ZipReader read"] = @benchmarkable ZipReader(read($(fname)))
+rfbench["zip_test read"] = @benchmarkable zip_test($(r))

test/runtests.jl

@@ -11,6 +11,7 @@ Random.seed!(1234)
 # @test Any[] == detect_ambiguities(Base, Core, ZipArchives)
 include("test_bytes2string.jl")
 include("test_simple-usage.jl")
+include("test_file-array.jl")
 include("test_filename-checks.jl")
 include("test_show.jl")
 include("test_writer.jl")

test/test_file-array.jl

@@ -0,0 +1,79 @@
+using ZipArchives: 
+    ZipWriter,
+    zip_newfile,
+    ZipReader,
+    zip_openentry,
+    zip_test
+
+using Test: @testset, @test
+
+@testset "Streaming one entry in a large archive file" begin
+    # create test file
+    fname = tempname()
+    ZipWriter(fname) do w
+        zip_newfile(w, "test/compressed.txt"; compress=true)
+        write(w, "Compressed \n Data \n"^1000)
+        zip_newfile(w, "test/uncompressed.txt")
+        write(w, "Uncompressed Data \n"^1000)
+    end
+
+    # Based on DiskArrays code suggested by Fabian Gans in 
+    # https://discourse.julialang.org/t/struggling-to-use-mmap-with-ziparchives/129839/19
+    struct FileArray <: AbstractVector{UInt8}
+        filename::String
+        offset::Int64
+        len::Int64
+    end
+    function FileArray(filename::String, offset::Int64=Int64(0))
+        len = filesize(filename)
+        len ≥ 0 || error("filesize of $(repr(filename)) is negative")
+        offset ≥ 0 || error("offset $(offset) is negative")
+        offset ≤ len || error("offset $(offset) is larger than the filesize $(len)")
+        FileArray(filename, offset, len-offset)
+    end
+    Base.size(s::FileArray) = (s.len,)
+    function Base.getindex(s::FileArray, i::Int)::UInt8
+        copyto!(zeros(UInt8, 1), Int64(1), s, Int64(i), Int64(1))[1]
+    end
+    function Base.view(s::FileArray, inds::UnitRange{Int64})::FileArray
+        checkbounds(s, inds)
+        FileArray(s.filename, s.offset + first(inds) - Int64(1), length(inds))
+    end
+    dest_types = if VERSION ≥ v"1.11"
+        (Vector{UInt8}, Memory{UInt8},)
+    else
+        (Vector{UInt8},)
+    end
+    for dest_type in dest_types
+        @eval begin
+            function Base.copyto!(dest::$dest_type, dstart::Int64, src::FileArray, sstart::Int64, n::Int64)
+                iszero(n) && return dest
+                n ≥ 0 || throw(ArgumentError("tried to copy n=$(n) elements, but n should be non-negative"))
+                checkbounds(dest, dstart)
+                checkbounds(src, sstart)
+                checkbounds(dest, dstart + n - Int64(1))
+                checkbounds(src, sstart + n - Int64(1))
+                open(src.filename) do io
+                    seek(io, src.offset + sstart - Int64(1))
+                    nb = readbytes!(io, view(dest, range(dstart; length=n)))
+                    nb == n || error("short read")
+                end
+                return dest
+            end
+        end
+    end
+
+    archive = ZipReader(FileArray(fname))
+
+    # validate all the checksums
+    zip_test(archive)
+
+    # `zip_openentry` is used to stream data from the entry as an `IO`.
+    # This works with both compressed and uncompressed entries.
+    zip_openentry(archive, "test/compressed.txt") do io
+        @test countlines(io) == 2000
+    end
+    zip_openentry(archive, "test/uncompressed.txt") do io
+        @test countlines(io) == 1000
+    end
+end