Nathan Evans' Nemesis of the Moment

Super fast way to extract width/height dimensions of PNG and JPEG images

Posted in .NET Framework, F# by Nathan B. Evans on April 17, 2015

Turns out that getting the width and height of an image file can be quite tricky if you don’t want to read the whole file into memory. The System.Drawing.Image type in .NET will read it all into memory; not good.

So I read the PNG and JPEG specifications and came up with this.

PNG was easy, as that has a static file header where the width and height are always stored in the same place. But JPEG is far far trickier since the header is built up of segments which are not in any particular order. There can be dozens of these header segments. There is always a particular segment called a Start Of Frame (SOF) which is the one that contains the width/height.

I’ve tried to build this as robustly and defensively as possible since I intend to use on both the server-side and on low memory mobile devices. It is also good at detecting invalid or malformed files and failing fast on those conditions.

It supports big and little endian architectures. And it supports memory streams, file streams and network streams i.e. both seekable and unseekable streams.

The JPEG implementation uses a little mutable state for performance and memory conservation reasons.

ImageSize for F#

/// Provides services to extract header or metadata information from image files of supported types.
module nbevans.Util.ImageSize =
    type DimensionsResult =
    | Success of Width : int * Height : int
    | Malformed of Information : string
    | NotSupported

    module private Utils =
        let ntoh buffer index length =
            if BitConverter.IsLittleEndian then Array.Reverse(buffer, index, length)

        let ntoh_int32 buffer index =
            BitConverter.ToInt32(ntoh buffer index 4, index)

        let ntoh_int16 buffer index =
            BitConverter.ToInt16(ntoh buffer index 2, index)

        type Stream with
            /// Advances the position of the stream by seeking by a specified offset from the current position.
            /// Unlike Seek(), this method is safe for network streams and other types of stream where seeking is not possible.
            /// For these such "unseekable" streams, data will be read instead and immediately discarded.
            member stream.Advance(offset) =
                if stream.CanSeek then
                    stream.Seek(int64 offset, SeekOrigin.Current)
                    let buffer = Array.zeroCreate offset
                    stream.Read(buffer, 0, offset) |> ignore

    module Png =
        /// Gets the width & height dimensions of a PNG image.
        let dimensions (sourceStream:Stream) =
            let signature = Array.zeroCreate 8
            if sourceStream.Read(signature, 0, 8) <> 8 || signature <> [| 137uy; 80uy; 78uy; 71uy; 13uy; 10uy; 26uy; 10uy |] then
                let chunk = Array.zeroCreate 8
                if sourceStream.Advance(8) <> 16L || sourceStream.Read(chunk, 0, 8) <> 8 then
                    Malformed "Expected chunk is not present."
                    Success (ntoh_int32 chunk 0, ntoh_int32 chunk 4)

    module Jpeg =
        module private Markers =
            // All data markers are 2 bytes, where the first byte is a 0xFF prefix.
            let Prefix = 0xFFuy
            // The first data marker (i.e. first 2 bytes of the file) of every JPEG is this.
            let SOI_StartOfImage = 0xD8uy
            // JPEG has lots of different internal encoding types, which are indicated with a SOF data marker.
            // There are many like baseline, progressive, sequential, differential and various combinations of these too.
            // Fortunately the width/height is present in the same position of all of these SOF headers.
            let SOFn_StartOfFrame = [| 0xC0uy; 0xC1uy; 0xC2uy; 0xC3uy; 0xC5uy; 0xC6uy; 0xC7uy; 0xC9uy; 0xCAuy; 0xCBuy; 0xCDuy; 0xCEuy; 0xCFuy |]
            let SOS_StartOfScan = 0xDAuy

        /// Gets the width & height dimensions of a JPEG image.
        let dimensions (sourceStream:Stream) =
            let signature = Array.zeroCreate 2
            if sourceStream.Read(signature, 0, 2) <> 2 || signature.[0] <> Prefix || signature.[1] <> SOI_StartOfImage then
                let mutable result = Option<DimensionsResult>.None
                let marker = Array.zeroCreate 4

                while result.IsNone do
                    result <-
                        if sourceStream.Read(marker, 0, 4) <> 4 then
                            Some <| Malformed "Next data marker header cannot be read."
                            if marker.[0] = Prefix && SOFn_StartOfFrame |> Array.exists ((=) marker.[1]) then
                                // Reuse the marker array as a new buffer, skip over the first byte in the payload (which contains "sample precision"),
                                // and read the 4 bytes that contain two 16-bit values of the width and height, respectively.
                                let buffer = marker
                                sourceStream.Advance(1) |> ignore
                                if sourceStream.Read(buffer, 0, 4) <> 4 then
                                    Some <| Malformed "SOF data marker payload cannot be read."
                                    let lines = int <| ntoh_int16 buffer 0
                                    let samplesPerLine = int <| ntoh_int16 buffer 2
                                    Some <| Success (samplesPerLine, lines)

                            else if marker.[0] = Prefix && marker.[1] = SOS_StartOfScan then
                                // If we've reached the SOS marker then we missed the SOF marker.
                                // That's pretty bizarre and suggests a corrupt JPEG, or at least an unsupported SOF marker.
                                Some <| Malformed "SOS data marker was encountered prematurely."

                            else if marker.[0] <> Prefix then
                                // All data markers identifiers are 2 bytes and the first byte must be 0xFF.
                                Some <| Malformed "Next data marker header is malformed."

                                // After the data marker identifier is a 2 byte length (inclusive) of the payload.
                                // We need this to let us skip over the markers/payloads that are not interesting.
                                let length = (int <| ntoh_int16 marker 2) - 2
                                sourceStream.Advance(length) |> ignore

                defaultArg result (Malformed "End of data markers encountered prematurely.")
Tagged with: , , , , ,

Super skinny XML document generation with F#

Posted in .NET Framework, F# by Nathan B. Evans on April 15, 2015

I needed to generate some simple XML documents, in memory, from some F# script. From my C# days I was already familiar with the System.Xml.Linq namespace, which I still quite like. But it wasn’t particularly clean to use from F#. So I wrote a really simple F# wrapper for some of its most commonly used features.

XmlToolkit for F#

module nbevans.Util.XmlToolkit
open System.Text
open System.Xml
open System.Xml.Linq
open System.IO

let XDeclaration version encoding standalone = XDeclaration(version, encoding, standalone)
let XLocalName localName namespaceName = XName.Get(localName, namespaceName)
let XName expandedName = XName.Get(expandedName)
let XDocument xdecl content = XDocument(xdecl, content |> (fun v -> v :> obj) |> Seq.toArray)
let XComment (value:string) = XComment(value) :> obj
let XElementNS localName namespaceName content = XElement(XLocalName localName namespaceName, content |> (fun v -> v :> obj) |> Seq.toArray) :> obj
let XElement expandedName content = XElement(XName expandedName, content |> (fun v -> v :> obj) |> Seq.toArray) :> obj
let XAttributeNS localName namespaceName value = XAttribute(XLocalName localName namespaceName, value) :> obj
let XAttribute expandedName value = XAttribute(XName expandedName, value) :> obj

type XDocument with
    /// Saves the XML document to a MemoryStream using UTF-8 encoding, indentation and character checking.
    member doc.Save() =
        let ms = new MemoryStream()
        use xtw = XmlWriter.Create(ms, XmlWriterSettings(Encoding = Encoding.UTF8, Indent = true, CheckCharacters = true))
        ms.Position <- 0L

The principle of this module is that it overrides the key type names like System.Xml.Linq.XElement and the others with F# functions that effectively provide the equivalent constructor behaviour but in a more functional signature. Then a XDocument type extension adds a useful Save() function (since the stock ones are so useless on their own). … and here is an example usage straight from my app (but hopefully you will get the idea):

let doc =
    XDocument (XDeclaration "1.0" "UTF-8" "yes") [
        XComment "This document was automatically generated by a configuration script."
        XElement "Metadata" [
            XElement "SystemMetadata" [
                XElement "ScannedBy" ["PCT"]
                XElement "GenerationDate" [DateTime.UtcNow.ToString("s")]
                XElement "IndexedBy" ["UNKNOWN"]
                XElement "IndexedOn" ["UNKNOWN"]
                XElement "FileName" [createPackageContentFileName cp.Id fileName]
                XElement "ScanInfo" [
                    XElement "NumberOfPagesScanned" [string formPdfPageCount]
                    XElement "IpAddress" ["UNKNOWN"]
                    XElement "MachineName" ["UNKNOWN"]
                    XElement "NumberOfBlankPages" ["0"]
            XElement "UserDefinedMetadata" [
                XElement "Address1" [defaultArg (gp.Fields.TryFind "property-address") "UNKNOWN"]
                XElement "Postcode" [defaultArg (gp.Fields.TryFind "property-postcode") "UNKNOWN"]
                XElement "Patchcode" ["1"]
                XElement "Reviewdate" [DateTime.UtcNow.AddYears(1).ToString("s")]

let ms = doc.Save()
// 'ms' at this point contains a System.IO.MemoryStream of the generated XML document.
// That was my use-case, but maybe you will want to adapt this code or the XmlToolkit module itself to use a different type of stream; perhaps a FileStream. I'm a firm believer in K.I.S.S to avoid over-engineering.
Tagged with: , , ,