Generate performance metrics across probability thresholds

threshold_perf() can take a set of class probability predictions and determine performance characteristics across different values of the probability threshold and any existing groups.

Usage

threshold_perf(.data, ...)

# S3 method for data.frame
threshold_perf(
  .data,
  truth,
  estimate,
  thresholds = NULL,
  na_rm = TRUE,
  event_level = "first",
  ...
)

Arguments

.data: A tibble, potentially grouped.
...: Currently unused.
truth: The column identifier for the true two-class results (that is a factor). This should be an unquoted column name.
estimate: The column identifier for the predicted class probabilities (that is a numeric). This should be an unquoted column name.
thresholds: A numeric vector of values for the probability threshold. If unspecified, a series of values between 0.5 and 1.0 are used. Note: if this argument is used, it must be named.
na_rm: A single logical: should missing data be removed?
event_level: A single string. Either "first" or "second" to specify which level of truth to consider as the "event".

Value

A tibble with columns: .threshold, .estimator, .metric, .estimate and any existing groups.

Details

Note that that the global option yardstick.event_first will be used to determine which level is the event of interest. For more details, see the Relevant level section of yardstick::sens().

The currently calculated metrics are:

yardstick::j_index()
yardstick::sens()
yardstick::spec()
distance = (1 - sens) ^ 2 + (1 - spec) ^ 2

Examples

library(dplyr)
data("segment_logistic")

# Set the threshold to 0.6
# > 0.6 = good
# < 0.6 = poor
threshold_perf(segment_logistic, Class, .pred_good, thresholds = 0.6)
#> # A tibble: 4 × 4
#>   .threshold .metric  .estimator .estimate
#>        <dbl> <chr>    <chr>          <dbl>
#> 1        0.6 sens     binary         0.639
#> 2        0.6 spec     binary         0.869
#> 3        0.6 j_index  binary         0.508
#> 4        0.6 distance binary         0.148

# Set the threshold to multiple values
thresholds <- seq(0.5, 0.9, by = 0.1)

segment_logistic %>%
  threshold_perf(Class, .pred_good, thresholds)
#> # A tibble: 20 × 4
#>    .threshold .metric  .estimator .estimate
#>         <dbl> <chr>    <chr>          <dbl>
#>  1        0.5 sens     binary         0.714
#>  2        0.6 sens     binary         0.639
#>  3        0.7 sens     binary         0.561
#>  4        0.8 sens     binary         0.451
#>  5        0.9 sens     binary         0.249
#>  6        0.5 spec     binary         0.825
#>  7        0.6 spec     binary         0.869
#>  8        0.7 spec     binary         0.911
#>  9        0.8 spec     binary         0.937
#> 10        0.9 spec     binary         0.977
#> 11        0.5 j_index  binary         0.539
#> 12        0.6 j_index  binary         0.508
#> 13        0.7 j_index  binary         0.472
#> 14        0.8 j_index  binary         0.388
#> 15        0.9 j_index  binary         0.226
#> 16        0.5 distance binary         0.112
#> 17        0.6 distance binary         0.148
#> 18        0.7 distance binary         0.201
#> 19        0.8 distance binary         0.306
#> 20        0.9 distance binary         0.565

# ---------------------------------------------------------------------------

# It works with grouped data frames as well
# Let's mock some resampled data
resamples <- 5

mock_resamples <- resamples %>%
  replicate(
    expr = sample_n(segment_logistic, 100, replace = TRUE),
    simplify = FALSE
  ) %>%
  bind_rows(.id = "resample")

resampled_threshold_perf <- mock_resamples %>%
  group_by(resample) %>%
  threshold_perf(Class, .pred_good, thresholds)

resampled_threshold_perf
#> # A tibble: 100 × 5
#>    resample .threshold .metric .estimator .estimate
#>    <chr>         <dbl> <chr>   <chr>          <dbl>
#>  1 1               0.5 sens    binary         0.674
#>  2 1               0.6 sens    binary         0.628
#>  3 1               0.7 sens    binary         0.512
#>  4 1               0.8 sens    binary         0.419
#>  5 1               0.9 sens    binary         0.233
#>  6 2               0.5 sens    binary         0.645
#>  7 2               0.6 sens    binary         0.516
#>  8 2               0.7 sens    binary         0.484
#>  9 2               0.8 sens    binary         0.419
#> 10 2               0.9 sens    binary         0.194
#> # … with 90 more rows

# Average over the resamples
resampled_threshold_perf %>%
  group_by(.metric, .threshold) %>%
  summarise(.estimate = mean(.estimate))
#> `summarise()` has grouped output by '.metric'. You can override using the
#> `.groups` argument.
#> # A tibble: 20 × 3
#> # Groups:   .metric [4]
#>    .metric  .threshold .estimate
#>    <chr>         <dbl>     <dbl>
#>  1 distance        0.5     0.102
#>  2 distance        0.6     0.141
#>  3 distance        0.7     0.210
#>  4 distance        0.8     0.357
#>  5 distance        0.9     0.599
#>  6 j_index         0.5     0.587
#>  7 j_index         0.6     0.548
#>  8 j_index         0.7     0.477
#>  9 j_index         0.8     0.367
#> 10 j_index         0.9     0.210
#> 11 sens            0.5     0.720
#> 12 sens            0.6     0.651
#> 13 sens            0.7     0.554
#> 14 sens            0.8     0.406
#> 15 sens            0.9     0.228
#> 16 spec            0.5     0.867
#> 17 spec            0.6     0.897
#> 18 spec            0.7     0.923
#> 19 spec            0.8     0.961
#> 20 spec            0.9     0.982