Let’s use drake to train and compare multiple models in a unified automated workflow.

Packages

First, we load our packages into a fresh R session.

library(drake)
library(keras)
library(tidyverse)
library(rsample)
library(recipes)
library(yardstick)

Functions

drake is R-focused and function-oriented. We create functions to preprocess the data,

prepare_recipe <- function(data) {
  data %>%
    training() %>%
    recipe(Churn ~ .) %>%
    step_rm(customerID) %>%
    step_naomit(all_outcomes(), all_predictors()) %>%
    step_discretize(tenure, options = list(cuts = 6)) %>%
    step_log(TotalCharges) %>%
    step_mutate(Churn = ifelse(Churn == "Yes", 1, 0)) %>%
    step_dummy(all_nominal(), -all_outcomes()) %>%
    step_center(all_predictors(), -all_outcomes()) %>%
    step_scale(all_predictors(), -all_outcomes()) %>%
    prep()
}

define a keras model,

define_model <- function(rec) {
  input_shape <- ncol(
    juice(rec, all_predictors(), composition = "matrix")
  )
  keras_model_sequential() %>%
    layer_dense(
      units = 16,
      kernel_initializer = "uniform",
      activation = "relu",
      input_shape = input_shape
    ) %>%
    layer_dropout(rate = 0.1) %>%
    layer_dense(
      units = 16,
      kernel_initializer = "uniform",
      activation = "relu"
    ) %>%
    layer_dropout(rate = 0.1) %>%
    layer_dense(
      units = 1,
      kernel_initializer = "uniform",
      activation = "sigmoid"
    )
}

train and serialize a model,

train_model <- function(data, rec, batch_size) {
  model <- define_model(rec)
  compile(
    model,
    optimizer = "adam",
    loss = "binary_crossentropy",
    metrics = c("accuracy")
  )
  x_train_tbl <- juice(
    rec,
    all_predictors(),
    composition = "matrix"
  )
  y_train_vec <- juice(rec, all_outcomes()) %>%
    pull()
  fit(
    object = model,
    x = x_train_tbl,
    y = y_train_vec,
    batch_size = batch_size,
    epochs = 35,
    validation_split = 0.30,
    verbose = 0
  )
  serialize_model(model)
}

compare the predictions of a serialized model against reality,

confusion_matrix <- function(data, rec, serialized_model) {
  model <- unserialize_model(serialized_model)
  testing_data <- bake(rec, testing(data))
  x_test_tbl <- testing_data %>%
    select(-Churn) %>%
    as.matrix()
  y_test_vec <- testing_data %>%
    select(Churn) %>%
    pull()
  yhat_keras_class_vec <- model %>%
    predict_classes(x_test_tbl) %>%
    as.factor() %>%
    fct_recode(yes = "1", no = "0")
  yhat_keras_prob_vec <-
    model %>%
    predict_proba(x_test_tbl) %>%
    as.vector()
  test_truth <- y_test_vec %>%
    as.factor() %>%
    fct_recode(yes = "1", no = "0")
  estimates_keras_tbl <- tibble(
    truth = test_truth,
    estimate = yhat_keras_class_vec,
    class_prob = yhat_keras_prob_vec
  )
  estimates_keras_tbl %>%
    conf_mat(truth, estimate)
}

and compare the performance of multiple models.

compare_models <- function(...) {
  batch_sizes <- match.call()[-1] %>%
    as.character() %>%
    gsub(pattern = "conf_", replacement = "")
  df <- map_df(list(...), summary) %>%
    filter(.metric %in% c("accuracy", "sens", "spec")) %>%
    mutate(
      batch_size = rep(batch_sizes, each = n() / length(batch_sizes))
    ) %>%
    rename(metric = .metric, estimate = .estimate)
  ggplot(df) +
    geom_line(
      aes(x = metric, y = estimate, color = batch_size, group = batch_size)
    ) +
    theme_gray(16)
}

Plan

Next, we define our workflow in a drake plan. We will prepare the data, train different models with different batch sizes, and compare the models in terms of performance.

batch_sizes <- c(16, 32)
plan <- drake_plan(
  data = read_csv(file_in("customer_churn.csv"), col_types = cols()) %>%
    initial_split(prop = 0.3),
  rec = prepare_recipe(data),
  model = target(
    train_model(data, rec, batch_size),
    transform = map(batch_size = !!batch_sizes)
  ),
  conf = target(
    confusion_matrix(data, rec, model),
    transform = map(model, .id = batch_size)
  ),
  comparison = target(
    compare_models(conf),
    transform = combine(conf)
  )
)

The plan is a data frame with the steps we are going to do.

plan

Dependency graph

The graph visualizes the dependency relationships among the steps of the workflow.

config <- drake_config(plan)
vis_drake_graph(config)

Run the models

Call make() to actually run the workflow.

make(plan)
target data
target rec
target model_16
target model_32
target conf_16
target conf_32
target comparison

Inspect the results

The two models performed about the same.

readd(comparison) # see also loadd()

Add models

Let’s try another batch size.

batch_sizes <- c(16, 32, 64)
plan <- drake_plan(
  data = read_csv(file_in("customer_churn.csv"), col_types = cols()) %>%
    initial_split(prop = 0.3),
  rec = prepare_recipe(data),
  model = target(
    train_model(data, rec, batch_size),
    transform = map(batch_size = !!batch_sizes)
  ),
  conf = target(
    confusion_matrix(data, rec, model),
    transform = map(model, .id = batch_size)
  ),
  comparison = target(
    compare_models(conf),
    transform = combine(conf)
  )
)

We already trained models with batch sizes 16 and 32, and their dependencies have not changed, so some of our work is already up to date.

config <- drake_config(plan)
vis_drake_graph(config) # see also outdated() and predict_runtime()

make() only trains the outdated or missing models and refreshes the post-processing. It skips the targets that are already up to date.

make(plan)
target model_64
target conf_64
target comparison

Inspect the results again

readd(comparison) # see also loadd()

Going forward, we can turn our attention to different tuning parameters and try to improve specificity.

Tips

LS0tCnRpdGxlOiAiQXV0b21hdGVkIHdvcmtmbG93IgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpgYGB7ciBzZXR1cCwgaW5jbHVkZSA9IEZBTFNFfQpsaWJyYXJ5KGRyYWtlKQpsaWJyYXJ5KGtlcmFzKQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShyc2FtcGxlKQpsaWJyYXJ5KHJlY2lwZXMpCmxpYnJhcnkoeWFyZHN0aWNrKQpvcHRpb25zKAogIGRyYWtlX21ha2VfbWVudSA9IEZBTFNFLAogIGRyYWtlX2NsZWFuX21lbnUgPSBGQUxTRSwKICB3YXJuUGFydGlhbE1hdGNoQXJncyA9IEZBTFNFLAogIGNyYXlvbi5lbmFibGVkID0gRkFMU0UsCiAgcmVhZHIuc2hvd19wcm9ncmVzcyA9IEZBTFNFCikKY2xlYW4oZGVzdHJveSA9IFRSVUUpCmtuaXRyOjpvcHRzX2NodW5rJHNldCgKICBjb2xsYXBzZSA9IFRSVUUsCiAgY29tbWVudCA9ICIjPiIKKQpgYGAKCkxldCdzIHVzZSBbYGRyYWtlYF0oaHR0cHM6Ly9naXRodWIuY29tL3JvcGVuc2NpL2RyYWtlKSB0byB0cmFpbiBhbmQgY29tcGFyZSBtdWx0aXBsZSBtb2RlbHMgaW4gYSB1bmlmaWVkIGF1dG9tYXRlZCB3b3JrZmxvdy4KCiMjIFBhY2thZ2VzCgpGaXJzdCwgd2UgbG9hZCBvdXIgcGFja2FnZXMgaW50byBhIGZyZXNoIFIgc2Vzc2lvbi4KCmBgYHtyfQpsaWJyYXJ5KGRyYWtlKQpsaWJyYXJ5KGtlcmFzKQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShyc2FtcGxlKQpsaWJyYXJ5KHJlY2lwZXMpCmxpYnJhcnkoeWFyZHN0aWNrKQpgYGAKCiMjIEZ1bmN0aW9ucwoKW2BkcmFrZWBdKGh0dHBzOi8vZ2l0aHViLmNvbS9yb3BlbnNjaS9kcmFrZSkgaXMgUi1mb2N1c2VkIGFuZCBmdW5jdGlvbi1vcmllbnRlZC4gV2UgY3JlYXRlIGZ1bmN0aW9ucyB0byBbcHJlcHJvY2VzcyB0aGUgZGF0YV0oaHR0cHM6Ly9naXRodWIuY29tL3RpZHltb2RlbHMvcmVjaXBlcyksCgpgYGB7cn0KcHJlcGFyZV9yZWNpcGUgPC0gZnVuY3Rpb24oZGF0YSkgewogIGRhdGEgJT4lCiAgICB0cmFpbmluZygpICU+JQogICAgcmVjaXBlKENodXJuIH4gLikgJT4lCiAgICBzdGVwX3JtKGN1c3RvbWVySUQpICU+JQogICAgc3RlcF9uYW9taXQoYWxsX291dGNvbWVzKCksIGFsbF9wcmVkaWN0b3JzKCkpICU+JQogICAgc3RlcF9kaXNjcmV0aXplKHRlbnVyZSwgb3B0aW9ucyA9IGxpc3QoY3V0cyA9IDYpKSAlPiUKICAgIHN0ZXBfbG9nKFRvdGFsQ2hhcmdlcykgJT4lCiAgICBzdGVwX211dGF0ZShDaHVybiA9IGlmZWxzZShDaHVybiA9PSAiWWVzIiwgMSwgMCkpICU+JQogICAgc3RlcF9kdW1teShhbGxfbm9taW5hbCgpLCAtYWxsX291dGNvbWVzKCkpICU+JQogICAgc3RlcF9jZW50ZXIoYWxsX3ByZWRpY3RvcnMoKSwgLWFsbF9vdXRjb21lcygpKSAlPiUKICAgIHN0ZXBfc2NhbGUoYWxsX3ByZWRpY3RvcnMoKSwgLWFsbF9vdXRjb21lcygpKSAlPiUKICAgIHByZXAoKQp9CmBgYAoKZGVmaW5lIGEgW2BrZXJhc2BdKGh0dHBzOi8vZ2l0aHViLmNvbS9yc3R1ZGlvL2tlcmFzKSBtb2RlbCwKCmBgYHtyfQpkZWZpbmVfbW9kZWwgPC0gZnVuY3Rpb24ocmVjKSB7CiAgaW5wdXRfc2hhcGUgPC0gbmNvbCgKICAgIGp1aWNlKHJlYywgYWxsX3ByZWRpY3RvcnMoKSwgY29tcG9zaXRpb24gPSAibWF0cml4IikKICApCiAga2VyYXNfbW9kZWxfc2VxdWVudGlhbCgpICU+JQogICAgbGF5ZXJfZGVuc2UoCiAgICAgIHVuaXRzID0gMTYsCiAgICAgIGtlcm5lbF9pbml0aWFsaXplciA9ICJ1bmlmb3JtIiwKICAgICAgYWN0aXZhdGlvbiA9ICJyZWx1IiwKICAgICAgaW5wdXRfc2hhcGUgPSBpbnB1dF9zaGFwZQogICAgKSAlPiUKICAgIGxheWVyX2Ryb3BvdXQocmF0ZSA9IDAuMSkgJT4lCiAgICBsYXllcl9kZW5zZSgKICAgICAgdW5pdHMgPSAxNiwKICAgICAga2VybmVsX2luaXRpYWxpemVyID0gInVuaWZvcm0iLAogICAgICBhY3RpdmF0aW9uID0gInJlbHUiCiAgICApICU+JQogICAgbGF5ZXJfZHJvcG91dChyYXRlID0gMC4xKSAlPiUKICAgIGxheWVyX2RlbnNlKAogICAgICB1bml0cyA9IDEsCiAgICAgIGtlcm5lbF9pbml0aWFsaXplciA9ICJ1bmlmb3JtIiwKICAgICAgYWN0aXZhdGlvbiA9ICJzaWdtb2lkIgogICAgKQp9CmBgYAoKdHJhaW4gYW5kIFtzZXJpYWxpemVdKGh0dHBzOi8vdGVuc29yZmxvdy5yc3R1ZGlvLmNvbS9rZXJhcy9yZWZlcmVuY2Uvc2VyaWFsaXplX21vZGVsLmh0bWwpIGEgbW9kZWwsCgoKYGBge3J9CnRyYWluX21vZGVsIDwtIGZ1bmN0aW9uKGRhdGEsIHJlYywgYmF0Y2hfc2l6ZSkgewogIG1vZGVsIDwtIGRlZmluZV9tb2RlbChyZWMpCiAgY29tcGlsZSgKICAgIG1vZGVsLAogICAgb3B0aW1pemVyID0gImFkYW0iLAogICAgbG9zcyA9ICJiaW5hcnlfY3Jvc3NlbnRyb3B5IiwKICAgIG1ldHJpY3MgPSBjKCJhY2N1cmFjeSIpCiAgKQogIHhfdHJhaW5fdGJsIDwtIGp1aWNlKAogICAgcmVjLAogICAgYWxsX3ByZWRpY3RvcnMoKSwKICAgIGNvbXBvc2l0aW9uID0gIm1hdHJpeCIKICApCiAgeV90cmFpbl92ZWMgPC0ganVpY2UocmVjLCBhbGxfb3V0Y29tZXMoKSkgJT4lCiAgICBwdWxsKCkKICBmaXQoCiAgICBvYmplY3QgPSBtb2RlbCwKICAgIHggPSB4X3RyYWluX3RibCwKICAgIHkgPSB5X3RyYWluX3ZlYywKICAgIGJhdGNoX3NpemUgPSBiYXRjaF9zaXplLAogICAgZXBvY2hzID0gMzUsCiAgICB2YWxpZGF0aW9uX3NwbGl0ID0gMC4zMCwKICAgIHZlcmJvc2UgPSAwCiAgKQogIHNlcmlhbGl6ZV9tb2RlbChtb2RlbCkKfQpgYGAKCmNvbXBhcmUgdGhlIHByZWRpY3Rpb25zIG9mIGEgW3NlcmlhbGl6ZWRdKGh0dHBzOi8vdGVuc29yZmxvdy5yc3R1ZGlvLmNvbS9rZXJhcy9yZWZlcmVuY2Uvc2VyaWFsaXplX21vZGVsLmh0bWwpIG1vZGVsIGFnYWluc3QgcmVhbGl0eSwKCmBgYHtyfQpjb25mdXNpb25fbWF0cml4IDwtIGZ1bmN0aW9uKGRhdGEsIHJlYywgc2VyaWFsaXplZF9tb2RlbCkgewogIG1vZGVsIDwtIHVuc2VyaWFsaXplX21vZGVsKHNlcmlhbGl6ZWRfbW9kZWwpCiAgdGVzdGluZ19kYXRhIDwtIGJha2UocmVjLCB0ZXN0aW5nKGRhdGEpKQogIHhfdGVzdF90YmwgPC0gdGVzdGluZ19kYXRhICU+JQogICAgc2VsZWN0KC1DaHVybikgJT4lCiAgICBhcy5tYXRyaXgoKQogIHlfdGVzdF92ZWMgPC0gdGVzdGluZ19kYXRhICU+JQogICAgc2VsZWN0KENodXJuKSAlPiUKICAgIHB1bGwoKQogIHloYXRfa2VyYXNfY2xhc3NfdmVjIDwtIG1vZGVsICU+JQogICAgcHJlZGljdF9jbGFzc2VzKHhfdGVzdF90YmwpICU+JQogICAgYXMuZmFjdG9yKCkgJT4lCiAgICBmY3RfcmVjb2RlKHllcyA9ICIxIiwgbm8gPSAiMCIpCiAgeWhhdF9rZXJhc19wcm9iX3ZlYyA8LQogICAgbW9kZWwgJT4lCiAgICBwcmVkaWN0X3Byb2JhKHhfdGVzdF90YmwpICU+JQogICAgYXMudmVjdG9yKCkKICB0ZXN0X3RydXRoIDwtIHlfdGVzdF92ZWMgJT4lCiAgICBhcy5mYWN0b3IoKSAlPiUKICAgIGZjdF9yZWNvZGUoeWVzID0gIjEiLCBubyA9ICIwIikKICBlc3RpbWF0ZXNfa2VyYXNfdGJsIDwtIHRpYmJsZSgKICAgIHRydXRoID0gdGVzdF90cnV0aCwKICAgIGVzdGltYXRlID0geWhhdF9rZXJhc19jbGFzc192ZWMsCiAgICBjbGFzc19wcm9iID0geWhhdF9rZXJhc19wcm9iX3ZlYwogICkKICBlc3RpbWF0ZXNfa2VyYXNfdGJsICU+JQogICAgY29uZl9tYXQodHJ1dGgsIGVzdGltYXRlKQp9CmBgYAoKYW5kIGNvbXBhcmUgdGhlIHBlcmZvcm1hbmNlIG9mIG11bHRpcGxlIG1vZGVscy4gCgpgYGB7cn0KY29tcGFyZV9tb2RlbHMgPC0gZnVuY3Rpb24oLi4uKSB7CiAgYmF0Y2hfc2l6ZXMgPC0gbWF0Y2guY2FsbCgpWy0xXSAlPiUKICAgIGFzLmNoYXJhY3RlcigpICU+JQogICAgZ3N1YihwYXR0ZXJuID0gImNvbmZfIiwgcmVwbGFjZW1lbnQgPSAiIikKICBkZiA8LSBtYXBfZGYobGlzdCguLi4pLCBzdW1tYXJ5KSAlPiUKICAgIGZpbHRlcigubWV0cmljICVpbiUgYygiYWNjdXJhY3kiLCAic2VucyIsICJzcGVjIikpICU+JQogICAgbXV0YXRlKAogICAgICBiYXRjaF9zaXplID0gcmVwKGJhdGNoX3NpemVzLCBlYWNoID0gbigpIC8gbGVuZ3RoKGJhdGNoX3NpemVzKSkKICAgICkgJT4lCiAgICByZW5hbWUobWV0cmljID0gLm1ldHJpYywgZXN0aW1hdGUgPSAuZXN0aW1hdGUpCiAgZ2dwbG90KGRmKSArCiAgICBnZW9tX2xpbmUoCiAgICAgIGFlcyh4ID0gbWV0cmljLCB5ID0gZXN0aW1hdGUsIGNvbG9yID0gYmF0Y2hfc2l6ZSwgZ3JvdXAgPSBiYXRjaF9zaXplKQogICAgKSArCiAgICB0aGVtZV9ncmF5KDE2KQp9CmBgYAoKIyMgUGxhbgoKTmV4dCwgd2UgZGVmaW5lIG91ciB3b3JrZmxvdyBpbiBhIFtgZHJha2VgIHBsYW5dKGh0dHBzOi8vcm9wZW5zY2lsYWJzLmdpdGh1Yi5pby9kcmFrZS1tYW51YWwvcGxhbnMuaHRtbCkuIFdlIHdpbGwgcHJlcGFyZSB0aGUgZGF0YSwgdHJhaW4gZGlmZmVyZW50IG1vZGVscyB3aXRoIGRpZmZlcmVudCBiYXRjaCBzaXplcywgYW5kIGNvbXBhcmUgdGhlIG1vZGVscyBpbiB0ZXJtcyBvZiBwZXJmb3JtYW5jZS4gCgpgYGB7cn0KYmF0Y2hfc2l6ZXMgPC0gYygxNiwgMzIpCgpwbGFuIDwtIGRyYWtlX3BsYW4oCiAgZGF0YSA9IHJlYWRfY3N2KGZpbGVfaW4oImN1c3RvbWVyX2NodXJuLmNzdiIpLCBjb2xfdHlwZXMgPSBjb2xzKCkpICU+JQogICAgaW5pdGlhbF9zcGxpdChwcm9wID0gMC4zKSwKICByZWMgPSBwcmVwYXJlX3JlY2lwZShkYXRhKSwKICBtb2RlbCA9IHRhcmdldCgKICAgIHRyYWluX21vZGVsKGRhdGEsIHJlYywgYmF0Y2hfc2l6ZSksCiAgICB0cmFuc2Zvcm0gPSBtYXAoYmF0Y2hfc2l6ZSA9ICEhYmF0Y2hfc2l6ZXMpCiAgKSwKICBjb25mID0gdGFyZ2V0KAogICAgY29uZnVzaW9uX21hdHJpeChkYXRhLCByZWMsIG1vZGVsKSwKICAgIHRyYW5zZm9ybSA9IG1hcChtb2RlbCwgLmlkID0gYmF0Y2hfc2l6ZSkKICApLAogIGNvbXBhcmlzb24gPSB0YXJnZXQoCiAgICBjb21wYXJlX21vZGVscyhjb25mKSwKICAgIHRyYW5zZm9ybSA9IGNvbWJpbmUoY29uZikKICApCikKYGBgCgpUaGUgcGxhbiBpcyBhIGRhdGEgZnJhbWUgd2l0aCB0aGUgc3RlcHMgd2UgYXJlIGdvaW5nIHRvIGRvLgoKYGBge3J9CnBsYW4KYGBgCgojIyBEZXBlbmRlbmN5IGdyYXBoCgpUaGUgZ3JhcGggdmlzdWFsaXplcyB0aGUgZGVwZW5kZW5jeSByZWxhdGlvbnNoaXBzIGFtb25nIHRoZSBzdGVwcyBvZiB0aGUgd29ya2Zsb3cuCgpgYGB7cn0KY29uZmlnIDwtIGRyYWtlX2NvbmZpZyhwbGFuKQp2aXNfZHJha2VfZ3JhcGgoY29uZmlnKQpgYGAKCiMjIFJ1biB0aGUgbW9kZWxzCgpDYWxsIFtgbWFrZSgpYF0oaHR0cHM6Ly9yb3BlbnNjaS5naXRodWIuaW8vZHJha2UvcmVmZXJlbmNlL21ha2UuaHRtbCkgdG8gYWN0dWFsbHkgcnVuIHRoZSB3b3JrZmxvdy4KCmBgYHtyfQptYWtlKHBsYW4pCmBgYAoKIyMgSW5zcGVjdCB0aGUgcmVzdWx0cwoKVGhlIHR3byBtb2RlbHMgcGVyZm9ybWVkIGFib3V0IHRoZSBzYW1lLgoKYGBge3J9CnJlYWRkKGNvbXBhcmlzb24pICMgc2VlIGFsc28gbG9hZGQoKQpgYGAKCiMjIEFkZCBtb2RlbHMKCkxldCdzIHRyeSBhbm90aGVyIGJhdGNoIHNpemUuCgpgYGB7cn0KYmF0Y2hfc2l6ZXMgPC0gYygxNiwgMzIsIDY0KQoKcGxhbiA8LSBkcmFrZV9wbGFuKAogIGRhdGEgPSByZWFkX2NzdihmaWxlX2luKCJjdXN0b21lcl9jaHVybi5jc3YiKSwgY29sX3R5cGVzID0gY29scygpKSAlPiUKICAgIGluaXRpYWxfc3BsaXQocHJvcCA9IDAuMyksCiAgcmVjID0gcHJlcGFyZV9yZWNpcGUoZGF0YSksCiAgbW9kZWwgPSB0YXJnZXQoCiAgICB0cmFpbl9tb2RlbChkYXRhLCByZWMsIGJhdGNoX3NpemUpLAogICAgdHJhbnNmb3JtID0gbWFwKGJhdGNoX3NpemUgPSAhIWJhdGNoX3NpemVzKQogICksCiAgY29uZiA9IHRhcmdldCgKICAgIGNvbmZ1c2lvbl9tYXRyaXgoZGF0YSwgcmVjLCBtb2RlbCksCiAgICB0cmFuc2Zvcm0gPSBtYXAobW9kZWwsIC5pZCA9IGJhdGNoX3NpemUpCiAgKSwKICBjb21wYXJpc29uID0gdGFyZ2V0KAogICAgY29tcGFyZV9tb2RlbHMoY29uZiksCiAgICB0cmFuc2Zvcm0gPSBjb21iaW5lKGNvbmYpCiAgKQopCmBgYAoKV2UgYWxyZWFkeSB0cmFpbmVkIG1vZGVscyB3aXRoIGJhdGNoIHNpemVzIDE2IGFuZCAzMiwgYW5kIHRoZWlyIGRlcGVuZGVuY2llcyBoYXZlIG5vdCBjaGFuZ2VkLCBzbyBzb21lIG9mIG91ciB3b3JrIGlzIGFscmVhZHkgdXAgdG8gZGF0ZS4KCmBgYHtyfQpjb25maWcgPC0gZHJha2VfY29uZmlnKHBsYW4pCnZpc19kcmFrZV9ncmFwaChjb25maWcpICMgc2VlIGFsc28gb3V0ZGF0ZWQoKSBhbmQgcHJlZGljdF9ydW50aW1lKCkKYGBgCgpbYG1ha2UoKWBdKGh0dHBzOi8vcm9wZW5zY2kuZ2l0aHViLmlvL2RyYWtlL3JlZmVyZW5jZS9tYWtlLmh0bWwpIG9ubHkgdHJhaW5zIHRoZSBvdXRkYXRlZCBvciBtaXNzaW5nIG1vZGVscyBhbmQgcmVmcmVzaGVzIHRoZSBwb3N0LXByb2Nlc3NpbmcuIEl0IHNraXBzIHRoZSB0YXJnZXRzIHRoYXQgYXJlIGFscmVhZHkgdXAgdG8gZGF0ZS4KCgpgYGB7cn0KbWFrZShwbGFuKQpgYGAKCiMjIEluc3BlY3QgdGhlIHJlc3VsdHMgYWdhaW4KCmBgYHtyfQpyZWFkZChjb21wYXJpc29uKSAjIHNlZSBhbHNvIGxvYWRkKCkKYGBgCgpHb2luZyBmb3J3YXJkLCB3ZSBjYW4gdHVybiBvdXIgYXR0ZW50aW9uIHRvIGRpZmZlcmVudCB0dW5pbmcgcGFyYW1ldGVycyBhbmQgdHJ5IHRvIGltcHJvdmUgc3BlY2lmaWNpdHkuCgojIyBUaXBzCgotIFRvIHNhdmUgdGhpcyBjb2RlIGluIHdlbGwtb3JnYW5pemVkIFIgc2NyaXB0cywgc2VlIHRoZSBbZ3VpZGFuY2Ugb24gcGVyc2lzdGVudCBgZHJha2VgLXBvd2VyZWQgcHJvamVjdHNdKGh0dHBzOi8vcm9wZW5zY2lsYWJzLmdpdGh1Yi5pby9kcmFrZS1tYW51YWwvcHJvamVjdHMuaHRtbCkuCi0gW2BkcmFrZWBdKGh0dHBzOi8vZ2l0aHViLmNvbS9yb3BlbnNjaS9kcmFrZSkgaGFzIFtidWlsdC1pbiBkaXN0cmlidXRlZCBjb21wdXRpbmcgc3VwcG9ydF0oaHR0cHM6Ly9yb3BlbnNjaWxhYnMuZ2l0aHViLmlvL2RyYWtlLW1hbnVhbC9ocGMuaHRtbCkgdGhhdCBsZXRzIHlvdSBmaXQgbXVsdGlwbGUgbW9kZWxzIGluIHBhcmFsbGVsLgoKYGBge3IsIGVjaG8gPSBGQUxTRX0KY2xlYW4oZGVzdHJveSA9IFRSVUUpCmBgYAo=