8 Running test cases
Using package wyz.code.offensiveProgramming, you have the opportunity to define test cases and to embed them in your class definition, to ease retrieval and reuse.
8.1 Reusing defined test case definitions
Doing so, allows to get following benefits
- discover defined test case definitions
- run of any test case definition
- get interactively the R code of a test case, allowing you to play with it, manually, when needed
- get contextual results from the test case runs
8.2 Embedding test cases in class definition
This is accomplished easily. You just have to declare a variable named testCaseDefinitions, and provide its content, that is a data.table. Content could be partial or complete depending of your goals. Spectrum of provided tests cases is as you desire it to be, as shallow or deep as needed.
The data.table must hold following columns and content
- function_name, a vector of strings, each being the name of the function to test,
- standard_evaluation, a vector of strings, where values are taken from set {‘correct’, ‘erroneous’, ‘failure’}
- type_checking_enforcement, , a vector of strings, where values are taken from set {‘correct’, ‘erroneous’, ‘failure’}
- test case definitions, that is a list of TestCaseDefinition objects.
Correct implies right type and right result. Erroneous implies right type and wrong result. Failure implies wrong type. To get more details about syntax, please refer to manual page of Definitions.
8.2.1 A simple case
source(system.file('code-samples/both-defs/good/partial/AdditionTCFIP.R', package = 'wyz.code.offensiveProgramming'))
AdditionTCFIP
#> function ()
#> {
#> self <- environment()
#> class(self) <- append("AdditionTCFIP", class(self))
#> addNumeric <- function(x_n, y_n) x_n + y_n
#> addDouble <- function(x_d, y_d = 0, ...) x_d + y_d + ...
#> addInteger <- function(x_i, y_i) x_i + y_i
#> divideByZero <- function(x_n) x_n/0
#> generateWarning <- function(x_ = 8L) 1:3 + 1:7 + x_
#> generateError <- function() stop("generated error")
#> function_return_types <- data.table(function_name = c("addNumeric",
#> "addDouble", "addInteger", "divideByZero", "generateWarning",
#> "generateError"), return_value = c("x_n", "x_d", "x_i",
#> "x_d", "x_w", "x_er"))
#> test_case_definitions <- data.table(function_name = c("addInteger",
#> "divideByZero", "divideByZero", "generateWarning", "generateError"),
#> standard_evaluation = c("correct", "correct", "correct",
#> "correct", "failure"), type_checking_enforcement = c("erroneous",
#> "correct", "correct", "correct", "correct"), test_case = list(TestCaseDefinition(list(34L,
#> 44.5), 78L, "sum 1 integer and 1 double"), TestCaseDefinition(list(1),
#> Inf, "1 / 0"), TestCaseDefinition(list(0), NaN, "0 / 0"),
#> TestCaseDefinition(list(0), 1:3 + 1:7, "generate warning"),
#> TestCaseDefinition(list(), NA, "generate error")))
#> self
#> }Just five test cases to test 5 function with various parameters.
8.2.2 A more complex case
AdditionTCFIG1
#> function ()
#> {
#> self <- environment()
#> class(self) <- append("AdditionTCFIG1", class(self))
#> addDouble <- function(x_d, y_d) x_d + y_d
#> addInteger <- function(x_i, y_i) x_i + y_i
#> addMultiDouble <- function(...) as.double(sum(..., na.rm = TRUE))
#> addMultiInteger <- function(x_i, ...) x_i + sum(..., na.rm = TRUE)
#> divideByZero <- function(x_n) x_n/0
#> generateWarning <- function(x_) 1:3 + 1:7
#> generateError <- function() stop("generated error")
#> function_return_types <- data.table(function_name = c("addDouble",
#> "addInteger", "addMultiDouble", "divideByZero", "addMultiInteger",
#> "generateWarning", "generateError"), return_value = c("x_d",
#> "x_i", "x_d", "x_d", "x_i", "x_w", "x_er"))
#> test_case_definitions <- data.table(function_name = c(rep("addDouble",
#> 9), rep("addInteger", 9), rep("divideByZero", 3), "generateWarning",
#> "generateError", rep("addMultiDouble", 3), rep("addMultiInteger",
#> 3)), standard_evaluation = c(rep("correct", 5), "erroneous",
#> rep("correct", 7), "erroneous", rep("correct", 8), "failure",
#> rep("correct", 6)), type_checking_enforcement = c(rep("correct",
#> 5), "erroneous", rep("failure", 3), rep("correct", 4),
#> rep("failure", 5), rep("correct", 3), "correct", "failure",
#> "correct", "correct", "correct", "correct", "failure",
#> "correct"), test_case = list(TestCaseDefinition(list(as.double(34L),
#> 44.5), 78.5, "sum 2 doubles"), TestCaseDefinition(list(34,
#> NA_real_), NA_real_, "sum 1 double and 1 NA_real_"),
#> TestCaseDefinition(list(NA_real_, NA_real_), NA_real_,
#> "sum 2 NA_real_"), TestCaseDefinition(list(NaN, NaN),
#> NaN, "sum 2 NAN"), TestCaseDefinition(list(Inf, Inf),
#> Inf, "sum 2 Inf"), TestCaseDefinition(list(as.integer(34.7),
#> as.integer(44.9)), 80, "sum 2 as.integers confused with sum of rounded value as expectation"),
#> TestCaseDefinition(list(34L, 44.5), 78.5, "sum of 1 integer and 1 double"),
#> TestCaseDefinition(list(34, NA_integer_), NA_real_, "sum of 1 integer and 1 double"),
#> TestCaseDefinition(list(NA, NA), NA, "sum 2 NA"), TestCaseDefinition(list(34L,
#> as.integer(44.5)), 78L, "sum 2 integers"), TestCaseDefinition(list(34L,
#> NA_integer_), NA_integer_, "sum 1 integer and 1 NA_integer"),
#> TestCaseDefinition(list(NA_integer_, NA_integer_), NA_integer_,
#> "sum 2 NA_integer"), TestCaseDefinition(list(as.integer("45.654"),
#> 44L), 89L, "sum a converted string with one integer"),
#> TestCaseDefinition(list(34L, 44.5), 78L, "sum 1 integer and 1 double"),
#> TestCaseDefinition(list(34L, Inf), Inf, "sum 1 integer and 1 Inf"),
#> TestCaseDefinition(list(34L, NaN), NaN, "sum 1 integer and 1 NAN"),
#> TestCaseDefinition(list(34L, NA), NA, "sum 1 integer and 1 NA"),
#> TestCaseDefinition(list(c(34L, 35L), 44L), c(78L, 79L),
#> "sum a vector of 2 integers with 1 integer"), TestCaseDefinition(list(1),
#> Inf, "1 / 0"), TestCaseDefinition(list(-1), -Inf,
#> "-1 / 0"), TestCaseDefinition(list(0), NaN, "0 / 0"),
#> TestCaseDefinition(list(0), 1:3 + 1:7, "generate warning"),
#> TestCaseDefinition(list(), NA, "generate error"), TestCaseDefinition(list(34L,
#> 44.5), 78.5, "sum of 1 integer and 1 double"), TestCaseDefinition(list(34,
#> 35L, 36L, NA_integer_), 105, "sum of 1 double, 2 integers and 1 NA_integer_"),
#> TestCaseDefinition(list(), 0, "sum of nothing"), TestCaseDefinition(list(34L,
#> 44L, 1L, 1L), 80L, "sum of 4 integers"), TestCaseDefinition(list(34L,
#> 35, 36, NA_integer_), 105, "sum of 1 integer, 2 doubles and 1 NA_integer_"),
#> TestCaseDefinition(list(34L), 34L, "sum of one integer and nothing")))
#> label <- "erroneous class instrumentation: test cases uses function divideByZero that is not instrumented for type checking enforcement"
#> self
#> }
#> <bytecode: 0x000000001b062cd0>Much more complete instrumentation with 29 test cases, various expected outputs, varying from evaluation model to consider.
8.3 Test case definitions verification
To verify test cases definitions you may use the low level function verifyTestCaseDefinitions or the higher level one named retrieveTestCaseDefinitions.
retrieveTestCaseDefinitions(AdditionTCFIG1())
#> function_name standard_evaluation type_checking_enforcement
#> 1: addDouble correct correct
#> 2: addDouble correct correct
#> 3: addDouble correct correct
#> 4: addDouble correct correct
#> 5: addDouble correct correct
#> 6: addDouble erroneous erroneous
#> 7: addDouble correct failure
#> 8: addDouble correct failure
#> 9: addDouble correct failure
#> 10: addInteger correct correct
#> 11: addInteger correct correct
#> 12: addInteger correct correct
#> 13: addInteger correct correct
#> 14: addInteger erroneous failure
#> 15: addInteger correct failure
#> 16: addInteger correct failure
#> 17: addInteger correct failure
#> 18: addInteger correct failure
#> 19: divideByZero correct correct
#> 20: divideByZero correct correct
#> 21: divideByZero correct correct
#> 22: generateWarning correct correct
#> 23: generateError failure failure
#> 24: addMultiDouble correct correct
#> 25: addMultiDouble correct correct
#> 26: addMultiDouble correct correct
#> 27: addMultiInteger correct correct
#> 28: addMultiInteger correct failure
#> 29: addMultiInteger correct correct
#> function_name standard_evaluation type_checking_enforcement
#> test_case
#> 1: <TestCaseDefinition>
#> 2: <TestCaseDefinition>
#> 3: <TestCaseDefinition>
#> 4: <TestCaseDefinition>
#> 5: <TestCaseDefinition>
#> 6: <TestCaseDefinition>
#> 7: <TestCaseDefinition>
#> 8: <TestCaseDefinition>
#> 9: <TestCaseDefinition>
#> 10: <TestCaseDefinition>
#> 11: <TestCaseDefinition>
#> 12: <TestCaseDefinition>
#> 13: <TestCaseDefinition>
#> 14: <TestCaseDefinition>
#> 15: <TestCaseDefinition>
#> 16: <TestCaseDefinition>
#> 17: <TestCaseDefinition>
#> 18: <TestCaseDefinition>
#> 19: <TestCaseDefinition>
#> 20: <TestCaseDefinition>
#> 21: <TestCaseDefinition>
#> 22: <TestCaseDefinition>
#> 23: <TestCaseDefinition>
#> 24: <TestCaseDefinition>
#> 25: <TestCaseDefinition>
#> 26: <TestCaseDefinition>
#> 27: <TestCaseDefinition>
#> 28: <TestCaseDefinition>
#> 29: <TestCaseDefinition>
#> test_case8.4 Discovering test cases descriptions
Any R object instrumented with test case definitions allows for defined test case definitions discovery.
retrieveTestCaseDescriptions(AdditionTCFIG1())
#> function_name
#> 1: addDouble
#> 2: addDouble
#> 3: addDouble
#> 4: addDouble
#> 5: addDouble
#> 6: addDouble
#> 7: addDouble
#> 8: addDouble
#> 9: addDouble
#> 10: addInteger
#> 11: addInteger
#> 12: addInteger
#> 13: addInteger
#> 14: addInteger
#> 15: addInteger
#> 16: addInteger
#> 17: addInteger
#> 18: addInteger
#> 19: divideByZero
#> 20: divideByZero
#> 21: divideByZero
#> 22: generateWarning
#> 23: generateError
#> 24: addMultiDouble
#> 25: addMultiDouble
#> 26: addMultiDouble
#> 27: addMultiInteger
#> 28: addMultiInteger
#> 29: addMultiInteger
#> function_name
#> description
#> 1: sum 2 doubles
#> 2: sum 1 double and 1 NA_real_
#> 3: sum 2 NA_real_
#> 4: sum 2 NAN
#> 5: sum 2 Inf
#> 6: sum 2 as.integers confused with sum of rounded value as expectation
#> 7: sum of 1 integer and 1 double
#> 8: sum of 1 integer and 1 double
#> 9: sum 2 NA
#> 10: sum 2 integers
#> 11: sum 1 integer and 1 NA_integer
#> 12: sum 2 NA_integer
#> 13: sum a converted string with one integer
#> 14: sum 1 integer and 1 double
#> 15: sum 1 integer and 1 Inf
#> 16: sum 1 integer and 1 NAN
#> 17: sum 1 integer and 1 NA
#> 18: sum a vector of 2 integers with 1 integer
#> 19: 1 / 0
#> 20: -1 / 0
#> 21: 0 / 0
#> 22: generate warning
#> 23: generate error
#> 24: sum of 1 integer and 1 double
#> 25: sum of 1 double, 2 integers and 1 NA_integer_
#> 26: sum of nothing
#> 27: sum of 4 integers
#> 28: sum of 1 integer, 2 doubles and 1 NA_integer_
#> 29: sum of one integer and nothing
#> description8.5 Run a test case
To run a test case, you may use the package function runTestCase.
runTestCase(AdditionTCFIG1(), 4, EvaluationMode(defineEvaluationModes()[3]))
#> $raw
#> $raw$addDouble
#> $raw$addDouble$status
#> [1] TRUE
#>
#> $raw$addDouble$value
#> [1] NaN
#>
#> $raw$addDouble$mode
#> [1] "type_checking_enforcement"
#>
#> $raw$addDouble$function_return_check
#> [1] TRUE
#>
#> $raw$addDouble$parameter_check
#> [1] TRUE
#>
#> $raw$addDouble$parameter_type_checks
#> parameter_name parameter_value validity message
#> 1: x_d NaN TRUE good type in values
#> 2: y_d NaN TRUE good type in values
#>
#> $raw$addDouble$function_return_type_check
#> parameter_name parameter_value validity message
#> 1: x_d NaN TRUE good type in values
#>
#> $raw$addDouble$index
#> [1] 4
#>
#> $raw$addDouble$value_check
#> [1] TRUE
#>
#> $raw$addDouble$expected_evaluation
#> [1] "correct"
#>
#> $raw$addDouble$execution_evaluation
#> [1] "correct"
#>
#> $raw$addDouble$failure_origin
#> [1] NA
#>
#>
#>
#> $synthesis
#> status mode index value_check
#> 1: TRUE type_checking_enforcement 4 TRUE
#> function_return_check parameter_check expected_evaluation
#> 1: TRUE TRUE correct
#> execution_evaluation failure_origin
#> 1: correct <NA>This runs the test number 4. Result has two parts. A raw part, that holds the intermediate computation results, and a synthesis part that is a data.table provided to ease result interpretation.
You can provide a vector instead of a single test number if you want to run several use test cases in one call.
runTestCase(AdditionTCFIG1(), 12:17, EvaluationMode(defineEvaluationModes()[3]))
#> $raw
#> $raw$addInteger
#> $raw$addInteger$status
#> [1] TRUE
#>
#> $raw$addInteger$value
#> [1] NA
#>
#> $raw$addInteger$mode
#> [1] "type_checking_enforcement"
#>
#> $raw$addInteger$function_return_check
#> [1] TRUE
#>
#> $raw$addInteger$parameter_check
#> [1] TRUE
#>
#> $raw$addInteger$parameter_type_checks
#> parameter_name parameter_value validity message
#> 1: x_i NA TRUE good type in values
#> 2: y_i NA TRUE good type in values
#>
#> $raw$addInteger$function_return_type_check
#> parameter_name parameter_value validity message
#> 1: x_i NA TRUE good type in values
#>
#> $raw$addInteger$index
#> [1] 12
#>
#> $raw$addInteger$value_check
#> [1] TRUE
#>
#> $raw$addInteger$expected_evaluation
#> [1] "correct"
#>
#> $raw$addInteger$execution_evaluation
#> [1] "correct"
#>
#> $raw$addInteger$failure_origin
#> [1] NA
#>
#>
#> $raw$addInteger
#> $raw$addInteger$status
#> [1] TRUE
#>
#> $raw$addInteger$value
#> [1] 89
#>
#> $raw$addInteger$mode
#> [1] "type_checking_enforcement"
#>
#> $raw$addInteger$function_return_check
#> [1] TRUE
#>
#> $raw$addInteger$parameter_check
#> [1] TRUE
#>
#> $raw$addInteger$parameter_type_checks
#> parameter_name parameter_value validity message
#> 1: x_i 45 TRUE good type in values
#> 2: y_i 44 TRUE good type in values
#>
#> $raw$addInteger$function_return_type_check
#> parameter_name parameter_value validity message
#> 1: x_i 89 TRUE good type in values
#>
#> $raw$addInteger$index
#> [1] 13
#>
#> $raw$addInteger$value_check
#> [1] TRUE
#>
#> $raw$addInteger$expected_evaluation
#> [1] "correct"
#>
#> $raw$addInteger$execution_evaluation
#> [1] "correct"
#>
#> $raw$addInteger$failure_origin
#> [1] NA
#>
#>
#> $raw$addInteger
#> $raw$addInteger$status
#> [1] FALSE
#>
#> $raw$addInteger$value
#> [1] 78.5
#>
#> $raw$addInteger$mode
#> [1] "type_checking_enforcement"
#>
#> $raw$addInteger$function_return_check
#> [1] FALSE
#>
#> $raw$addInteger$parameter_check
#> [1] FALSE
#>
#> $raw$addInteger$parameter_type_checks
#> parameter_name parameter_value validity message
#> 1: x_i 34 TRUE good type in values
#> 2: y_i 44.5 FALSE wrong type in values
#>
#> $raw$addInteger$function_return_type_check
#> parameter_name parameter_value validity message
#> 1: x_i 78.5 FALSE wrong type in values
#>
#> $raw$addInteger$index
#> [1] 14
#>
#> $raw$addInteger$value_check
#> [1] FALSE
#>
#> $raw$addInteger$expected_evaluation
#> [1] "failure"
#>
#> $raw$addInteger$execution_evaluation
#> [1] "failure"
#>
#> $raw$addInteger$failure_origin
#> [1] "function return type check, parameter check"
#>
#>
#> $raw$addInteger
#> $raw$addInteger$status
#> [1] FALSE
#>
#> $raw$addInteger$value
#> [1] Inf
#>
#> $raw$addInteger$mode
#> [1] "type_checking_enforcement"
#>
#> $raw$addInteger$function_return_check
#> [1] FALSE
#>
#> $raw$addInteger$parameter_check
#> [1] FALSE
#>
#> $raw$addInteger$parameter_type_checks
#> parameter_name parameter_value validity message
#> 1: x_i 34 TRUE good type in values
#> 2: y_i Inf FALSE wrong type in values
#>
#> $raw$addInteger$function_return_type_check
#> parameter_name parameter_value validity message
#> 1: x_i Inf FALSE wrong type in values
#>
#> $raw$addInteger$index
#> [1] 15
#>
#> $raw$addInteger$value_check
#> [1] TRUE
#>
#> $raw$addInteger$expected_evaluation
#> [1] "failure"
#>
#> $raw$addInteger$execution_evaluation
#> [1] "failure"
#>
#> $raw$addInteger$failure_origin
#> [1] "function return type check, parameter check"
#>
#>
#> $raw$addInteger
#> $raw$addInteger$status
#> [1] FALSE
#>
#> $raw$addInteger$value
#> [1] NaN
#>
#> $raw$addInteger$mode
#> [1] "type_checking_enforcement"
#>
#> $raw$addInteger$function_return_check
#> [1] FALSE
#>
#> $raw$addInteger$parameter_check
#> [1] FALSE
#>
#> $raw$addInteger$parameter_type_checks
#> parameter_name parameter_value validity message
#> 1: x_i 34 TRUE good type in values
#> 2: y_i NaN FALSE wrong type in values
#>
#> $raw$addInteger$function_return_type_check
#> parameter_name parameter_value validity message
#> 1: x_i NaN FALSE wrong type in values
#>
#> $raw$addInteger$index
#> [1] 16
#>
#> $raw$addInteger$value_check
#> [1] TRUE
#>
#> $raw$addInteger$expected_evaluation
#> [1] "failure"
#>
#> $raw$addInteger$execution_evaluation
#> [1] "failure"
#>
#> $raw$addInteger$failure_origin
#> [1] "function return type check, parameter check"
#>
#>
#> $raw$addInteger
#> $raw$addInteger$status
#> [1] FALSE
#>
#> $raw$addInteger$value
#> [1] NA
#>
#> $raw$addInteger$mode
#> [1] "type_checking_enforcement"
#>
#> $raw$addInteger$function_return_check
#> [1] TRUE
#>
#> $raw$addInteger$parameter_check
#> [1] FALSE
#>
#> $raw$addInteger$parameter_type_checks
#> parameter_name parameter_value validity message
#> 1: x_i 34 TRUE good type in values
#> 2: y_i NA FALSE wrong type in values
#>
#> $raw$addInteger$function_return_type_check
#> parameter_name parameter_value validity message
#> 1: x_i NA TRUE good type in values
#>
#> $raw$addInteger$index
#> [1] 17
#>
#> $raw$addInteger$value_check
#> [1] TRUE
#>
#> $raw$addInteger$expected_evaluation
#> [1] "failure"
#>
#> $raw$addInteger$execution_evaluation
#> [1] "failure"
#>
#> $raw$addInteger$failure_origin
#> [1] "parameter check"
#>
#>
#>
#> $synthesis
#> status mode index value_check
#> 1: TRUE type_checking_enforcement 12 TRUE
#> 2: TRUE type_checking_enforcement 13 TRUE
#> 3: FALSE type_checking_enforcement 14 FALSE
#> 4: FALSE type_checking_enforcement 15 TRUE
#> 5: FALSE type_checking_enforcement 16 TRUE
#> 6: FALSE type_checking_enforcement 17 TRUE
#> function_return_check parameter_check expected_evaluation
#> 1: TRUE TRUE correct
#> 2: TRUE TRUE correct
#> 3: FALSE FALSE failure
#> 4: FALSE FALSE failure
#> 5: FALSE FALSE failure
#> 6: TRUE FALSE failure
#> execution_evaluation failure_origin
#> 1: correct <NA>
#> 2: correct <NA>
#> 3: failure function return type check, parameter check
#> 4: failure function return type check, parameter check
#> 5: failure function return type check, parameter check
#> 6: failure parameter checkLooking at synthesis, you will discover that test 17 fails under chosen evaluation mode, and therefore should require a fix. Here looking at raw results for test number 17, brings solution, that is about input parameter compliance. Provided values are double, whereas integers were expected.