Multiple items
val ( 5 add 3 gives 8 ) : unit -> 'a

Full name: index.( 5 add 3 gives 8 )

--------------------
val ( 5 add 3 gives 8 ) : unit -> 'a

Full name: index.( 5 add 3 gives 8 )
val actual : obj
val ( isPalindrome returns true for "kajak" ) : unit -> 'a

Full name: index.( isPalindrome returns true for "kajak" )
val x : int
val y : int
val ( customer is important if she has X subscriptions ) : unit -> 'a

Full name: index.( customer is important if she has X subscriptions )
val inputXml : string
val result : obj
val ( ignores disabled nuget feed from upstream ) : unit -> 'a

Full name: index.( ignores disabled nuget feed from upstream )
val upstream : obj
union case Option.None: Option<'T>
Multiple items
module Map

from Microsoft.FSharp.Collections

--------------------
type Map<'Key,'Value (requires comparison)> =
  interface IEnumerable
  interface IComparable
  interface IEnumerable<KeyValuePair<'Key,'Value>>
  interface ICollection<KeyValuePair<'Key,'Value>>
  interface IDictionary<'Key,'Value>
  new : elements:seq<'Key * 'Value> -> Map<'Key,'Value>
  member Add : key:'Key * value:'Value -> Map<'Key,'Value>
  member ContainsKey : key:'Key -> bool
  override Equals : obj -> bool
  member Remove : key:'Key -> Map<'Key,'Value>
  ...

Full name: Microsoft.FSharp.Collections.Map<_,_>

--------------------
new : elements:seq<'Key * 'Value> -> Map<'Key,'Value>
val ofList : elements:('Key * 'T) list -> Map<'Key,'T> (requires comparison)

Full name: Microsoft.FSharp.Collections.Map.ofList
val next : obj
val expected : obj
val overridden : obj
val ( add gives sum of two components ) : x:'a * y:'b * expected:'c -> 'd

Full name: index.( add gives sum of two components )
val x : 'a
val y : 'b
val expected : 'c
Multiple items
val ( add gives sum of two components ) : x:int * y:int -> 'a

Full name: index.( add gives sum of two components )

--------------------
val ( add gives sum of two components ) : x:'a * y:'b * expected:'c -> 'd

Full name: index.( add gives sum of two components )
val expected : int
val ( parsing a stringified JSON gives original result ) : original:'a -> bool

Full name: index.( parsing a stringified JSON gives original result )
val original : 'a
val stringified : obj
val parsed : 'a
val ( distinct is idemptotent ) : input:'a list -> bool

Full name: index.( distinct is idemptotent )
val input : 'a list
type 'T list = List<'T>

Full name: Microsoft.FSharp.Collections.list<_>
val firstTurn : obj
val secondTurn : obj
val ( optimised version really works ) : input:'a -> bool

Full name: index.( optimised version really works )
val input : 'a
val optimisedResult : obj
val plainResult : obj
val title : obj

Full name: index.title
val body : obj

Full name: index.body
val topic : obj

Full name: index.topic
val schema : obj

Full name: index.schema
val ( modifier XML conforms to schema ) : topic:'a -> 'b

Full name: index.( modifier XML conforms to schema )
val topic : 'a
val output : obj
val ( if text node under "b" element then richtext has bold ) : topic:'a -> 'b

Full name: index.( if text node under "b" element then richtext has bold )
val textNodes : seq<obj>
val richtexts : seq<obj>
module Seq

from Microsoft.FSharp.Collections
val zip : source1:seq<'T1> -> source2:seq<'T2> -> seq<'T1 * 'T2>

Full name: Microsoft.FSharp.Collections.Seq.zip
val filter : predicate:('T -> bool) -> source:seq<'T> -> seq<'T>

Full name: Microsoft.FSharp.Collections.Seq.filter
val fst : tuple:('T1 * 'T2) -> 'T1

Full name: Microsoft.FSharp.Core.Operators.fst
val snd : tuple:('T1 * 'T2) -> 'T2

Full name: Microsoft.FSharp.Core.Operators.snd

Property based testing with F#

Plan

  1. Standard approach
  2. Property based approach
  3. Working example
  4. How to come up with a good test
  5. How we use it in Phoenix

Credits: http://fsharpforfunandprofit.com/

Standard approach

or "what's wrong with unit tests?"

1. Correctness

"Unit tests have been compared with shining a flashlight into a dark room in search of a monster. Shine the light into the room and then into all the scary corners. It doesn't mean the room is monster free - just that the monster isn't standing where you've shined your flashlight."

Monsters

1: 
2: 
3: 
4: 

[<Fact>]
let ``5 add 3 gives 8``() =
    let actual = add 5 3
    Assert.Equal(8, actual)
  • How could add be implemented?
1: 
2: 
3: 
4: 

[<Fact>]
let ``isPalindrome returns true for "kajak"``() =
    let actual = isPalindrome "kajak"
    Assert.Equal(true, actual)
  • How certain are we that the function is correctly implemented?
  • How is null handled?
  • How does it work for input with even count of letters?

"Unit tests do not prove that a program runs correctly. Unit tests may at most tell that the program does not fail for specific cases."

"How many tests are enough?"

2. Arrange phase

(also known as fixture)

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 

[<Fact>]
let ``5 add 3 gives 8``() =
    // Arrange phase
    let x = 5
    let y = 3

    // Act phase
    let actual = add x y

    // Assert phase
    Assert.Equal(8, actual)

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 

[<Fact>]
let ``customer is important if she has X subscriptions``() =
    // Arrange phase
    let inputXml = 
        """<customer>
                <firstName>John</firstName>
                <lastName>Doe</lastName>
                <email>john.doe@example.com</com>
                <subscriptions>
                    <subscription>
                        <type>...
                ...
           </customer>
        """
    // Act phase
    let result = classifyCustomer inputXml

    // Assert phase
    Assert.True(result.IsImportantCustomer)

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 
20: 
21: 
22: 
23: 
24: 

[<Test>]
let ``ignores disabled nuget feed from upstream`` () =
    // Arrange
    let upstream = 
        { NugetConfig.Empty with
            PackageSources = 
                [ "MyGetDuality", ("https://www.myget.org/", None) ]
                |> Map.ofList }
    let next = 
        // "Config.xml" is a file created for the arrange part
        NugetConfig.GetConfigNode (FileInfo "Config.xml") 
        |> Trial.returnOrFail
    let expected =
        { PackageSources = 
            [ "nuget.org", ("https://www.nuget.org/api/v2/",None) ]
            |> Map.ofList
          PackageRestoreEnabled = true
          PackageRestoreAutomatic = true }

    // Act
    let overridden = NugetConfig.OverrideConfig upstream next

    // Assert
    overridden |> shouldEqual expected

Issues with standard approach

(unit tests)

  1. Correctness is not guaranteed
  2. Arrange phase can be overwhelming

Property based approach

Idea

  • Don't test for specific cases
  • Think about what properties your code should have
  • Test input / fixture is indeterministic - but it's always generated for a certain type
  • We manipulate the generator, and the library makes sure to provide arbitrary instances

http://fsharpforfunandprofit.com/posts/property-based-testing/

Building blocks

  1. Generator
  2. Shrinker

Generator

  • Library comes with predefined generators for primitive types and most basic language structures
  • The mechanism is extensible, i.e. we can create our own generators for certain cases
  • Given a generator and a seed, the library generates random instances of the desired type
  • A single test is repeated multiple times, each time with a slightly "bigger" input

Shrinker

  • If for some input a test fails, the input is being shrunk
  • If the shrunk input still causes the test to fail, it's being shrunk again
  • Process is repeated until minimal faulty input is found

Property based approach

solving issues with unit tests

  1. Arbitrary input can detect edge cases - Correctness
  2. Much simpler and consistent test fixture - Arrange

Working example

(a journey to Nirvana)

Revisited add - initial version

1: 
2: 
3: 
4: 

[<Fact>]
let ``5 add 3 gives 8``() =
    let actual = add 5 3
    Assert.Equal(8, actual)

Add - parametrized tests

1: 
2: 
3: 
4: 
5: 
6: 
7: 

[<Theory>]
[<InlineData(5, 3, 8)>]
[<InlineData(2, 2, 4)>]
[<InlineData(-4, 9, 5)>]
let ``add gives sum of two components``(x, y, expected) =
    let actual = add x y
    Assert.Equal(expected, actual)

Add - using AutoFixture library

1: 
2: 
3: 
4: 
5: 
6: 

[<Theory>]
[<AutoData>]
let ``add gives sum of two components``(x, y) =
    let expected = x + y
    let actual = add x y
    Assert.Equal(expected, actual)
  • run only once?
  • what about using the "+" operator?

Add - using Property Based Approach

Demo

How to come up with a good test

applicable to a more complex problem

Scott Wlaschin's list

  1. "Different paths, same destination"
  2. "There and back again"
  3. "Some things never change"
  4. "The more things change, the more they stay the same"
  5. "Solve a smaller problem first"
  6. "Hard to prove, easy to verify"
  7. "The test oracle"

http://fsharpforfunandprofit.com/posts/property-based-testing-2/

"There and back again"

1: 
2: 
3: 
4: 
5: 

[<Property>]
let ``parsing a stringified JSON gives original result`` (original: JSON) =
    let stringified = stringify original
    let parsed = parse stringified
    parsed = original

"The more things change, the more they stay the same"

1: 
2: 
3: 
4: 
5: 

[<Property>]
let ``distinct is idemptotent`` (input: list<'a>) =
    let firstTurn = distinct input
    let secondTurn = distinct firstTurn
    firstTurn = secondTurn

"The test oracle"

1: 
2: 
3: 
4: 
5: 

[<Property>]
let ``optimised version really works`` (input) =
    let optimisedResult = hiperFastConcurrentAlgorithm input
    let plainResult = simpleButSlowerAlgorithm input
    plainResult = optimisedResult

How we use it in Phoenix

Desktop publishing

pdf

Quark XPress

qxp

Adobe InDesign

indesign

Quark Stack

  • Quark Publishing Platform
  • Quark XML Author
  • Quark XPress + Quark XPress Server

Quark XPress Server

qxps

Quark XPress template + Modifier XML = PDF

Desktop publishing

pdf

Desktop publishing automation

automation

Modifier XML

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 

<LAYOUT>
    <PAGESEQUENCE MASTERREFERENCE="Main">
        <STORY>
            <PARAGRAPH PARASTYLE="m_header_1">
                <RICHTEXT>My first publication</RICHTEXT>
            </PARAGRAPH>
            <PARAGRAPH PARASTYLE="m_body">
                <RICHTEXT>Hello </RICHTEXT>
                <RICHTEXT BOLD="TRUE">world!</RICHTEXT>
            </PARAGRAPH>
        </STORY>
    </PAGESEQUENCE>
</LAYOUT>

Dita XML

(or rather Phoenix-Dita XML)

1: 
2: 
3: 
4: 
5: 
6: 

<topic>
    <title>My first publication</title>
    <body>
        <p>Hello <b>world!</b></p>
    </body>
</topic>

XSLT !!!

epic

XSLT can get as complex ...

complex_xslt

... as JavaScript ...

js_pyramid

... so it's good to have some tests

need_tests

Generator

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 

let title = gen {
    let! contents = contents
    return XElement("title", contents)
}

let body = gen {
    let! items = Gen.oneOf [ para; table; chart ] |> Gen.listOf
    return XElement("body", items)
}

let topic = gen {
    let! title = title
    let! body = body
    return XElement("topic", title, body)
}

Tests

1: 
2: 
3: 
4: 
5: 
6: 

let schema = XmlSchema.Parse "Modifier.xsd"

[<Property>]
let ``modifier XML conforms to schema`` (topic: XDocument) =
    let output = xsltTransform "topic.xslt" topic
    doesNotThrow (fun () -> schema.Validate output)
  • alternative -> schema-aware XSLT processor

Moar tests

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 

[<Property>]
let ``if text node under "b" element then richtext has bold`` (topic) =
    let output = xsltTransform "topic.xslt" topic
    let textNodes = topic  |> xpath "//text()"
    let richtexts = output |> xpath "//RICHTEXT"

    (textNodes, richtexts)
    ||> Seq.zip
    |> Seq.filter (fst >> xpath "ancestor::b")
    |> Seq.forAll (snd >> xpath "@BOLD = 'TRUE'")

Shrinker

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 

<topic>
    <title>My first publication</title>
    <body>
        <image href="unicorn.pdf">
        <p>Hello <b>world!</b></p>
        <table>
            <title>table</title>
            <tbody>
                <row>
                    <entry>aaa</entry>
                </row>
            </tbody>
        </table>
    </body>
</topic>

Original input

Shrinker

1: 
2: 
3: 
4: 
5: 
6: 

<topic>
    <title/>
    <body>
        <p><b>w</b></p>
    </body>
</topic>

Shrinked input

Using Property Based Testing for testing XSLT

Conclusions so far

  • (+) Automatic detection of edge cases
  • (+) Easier to maintain "Arrange" phase - everything is in Generator
  • (+) All tests rely on the same Generator - improves consistency
  • (+) Easy to find minimal faulty input thanks to Shrinker
  • (-) "Assert" phase is slightly more complex
  • (-) Error reasoning can sometimes get tricky

Questions?