Aprendizaje Automatico con Tensorflow y R

Instalar paquetes

pkgs <- c("keras", "lime", "rsample", "recipes", "yardstick", "corrr")
install.packages(pkgs)

tidyverse

http://tidyverse.org/

Parsed with column specification:
cols(
  .default = col_character(),
  SeniorCitizen = [32mcol_double()[39m,
  tenure = [32mcol_double()[39m,
  MonthlyCharges = [32mcol_double()[39m,
  TotalCharges = [32mcol_double()[39m
)
See spec(...) for full column specifications.

glimpse(datos_perdimiento)

rsample

https://tidymodels.github.io/rsample/

library(rsample)

set.seed(100)

separa_datos <- initial_split(
  datos_perdimiento, 
  prop = 0.3)

tbl_entrenar <- training(separa_datos)
tbl_prueba  <- testing(separa_datos)

recipes

https://tidymodels.github.io/recipes/

library(recipes)

receta <- tbl_entrenar %>%
  recipe(Churn ~ .) %>%
  step_rm(customerID) %>%
  step_naomit(all_outcomes(), all_predictors()) %>%
  step_discretize(tenure, options = list(cuts = 6)) %>%
  step_log(TotalCharges) %>%
  step_mutate(Churn = ifelse(Churn == "Yes", 1, 0)) %>%
  step_dummy(all_nominal(), -all_outcomes()) %>%
  step_center(all_predictors(), -all_outcomes()) %>%
  step_scale(all_predictors(), -all_outcomes()) %>%
  prep()

summary(receta)

save(receta, file = "../aplicacion/receta.RData")

x_tbl_entrenar <- receta %>% 
  juice(all_predictors(), composition = "matrix") 

y_vec_entrenar <- receta %>% 
  juice(all_outcomes()) %>% 
  pull()

baked_test <- bake(receta, tbl_prueba)

x_tbl_prueba <- baked_test %>%
  select(-Churn) %>%
  as.matrix()

y_vec_prueba <- baked_test %>%
  select(Churn) %>%
  pull()

Instalar Tensorflow & Keras

https://tensorflow.rstudio.com/tensorflow/articles/installation.html

https://tensorflow.rstudio.com/keras/#installation

library(tensorflow)
library(keras)

#install_tensorflow()
#install_keras()

Crear una red neural

model_keras <- keras_model_sequential() %>%
  layer_dense(
    units = 16, 
    kernel_initializer = "uniform", 
    activation = "relu", 
    input_shape = ncol(x_tbl_entrenar)) %>% 
  layer_dropout(rate = 0.1) %>%
  layer_dense(
    units = 16, 
    kernel_initializer = "uniform", 
    activation = "relu") %>% 
  layer_dropout(rate = 0.1) %>%
  layer_dense(
    units = 1, 
    kernel_initializer = "uniform", 
    activation = "sigmoid") %>% 
  compile(
    optimizer = 'adam',
    loss = 'binary_crossentropy',
    metrics = c('accuracy')
  )

model_keras

Model
___________________________________________________________________________________________________________
Layer (type)                                    Output Shape                              Param #          
===========================================================================================================
dense_1 (Dense)                                 (None, 16)                                576              
___________________________________________________________________________________________________________
dropout_1 (Dropout)                             (None, 16)                                0                
___________________________________________________________________________________________________________
dense_2 (Dense)                                 (None, 16)                                272              
___________________________________________________________________________________________________________
dropout_2 (Dropout)                             (None, 16)                                0                
___________________________________________________________________________________________________________
dense_3 (Dense)                                 (None, 1)                                 17               
===========================================================================================================
Total params: 865
Trainable params: 865
Non-trainable params: 0
___________________________________________________________________________________________________________

Correr el modelo

history <- fit(
  object = model_keras, 
  x = x_tbl_entrenar, 
  y = y_vec_entrenar,
  batch_size = 50, 
  epochs = 35,
  validation_split = 0.30,
  verbose = 0
)

2019-03-17 11:19:29.569656: I T:\src\github\tensorflow\tensorflow\core\platform\cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2

print(history)

Trained on 1,478 samples, validated on 634 samples (batch_size=50, epochs=35)
Final epoch (plot to see history):
val_loss: 0.4467
 val_acc: 0.7855
    loss: 0.4049
     acc: 0.8031

Ver los resultados

theme_set(theme_bw())

plot(history)

yhat_keras_class_vec <- model_keras %>%
  predict_classes(x_tbl_prueba) %>%
  as.factor() %>%
  fct_recode(yes = "1", no = "0")

yhat_keras_prob_vec  <- model_keras %>%
  predict_proba(x_tbl_prueba) %>%
  as.vector()

test_truth <- y_vec_prueba %>% 
  as.factor() %>% 
  fct_recode(yes = "1", no = "0")

estimates_keras_tbl <- tibble(
  truth      = test_truth,
  estimate   = yhat_keras_class_vec,
  class_prob = yhat_keras_prob_vec
)

estimates_keras_tbl

yardstick

https://tidymodels.github.io/yardstick/

yardstick is a package to estimate how well models are working using tidy data principals.

library(yardstick)

options(yardstick.event_first = FALSE)

estimates_keras_tbl %>% 
  conf_mat(truth, estimate)

          Truth
Prediction   no  yes
       no  3229  569
       yes  396  726

estimates_keras_tbl %>% 
  metrics(truth, estimate)


estimates_keras_tbl %>% 
  roc_auc(truth, class_prob)


estimates_keras_tbl %>%
  precision(truth, estimate) %>%
  bind_rows(
    estimates_keras_tbl %>% 
      recall(truth, estimate) 
  )


estimates_keras_tbl %>% 
  f_meas(truth, estimate, beta = 1)

lime

https://github.com/thomasp85/lime

library(lime)

model_type.keras.engine.sequential.Sequential <- function(x, ...) {
  "classification"
}

predict_model.keras.engine.sequential.Sequential <- function(x, newdata, type, ...) {
  pred <- predict_proba(object = x, x = as.matrix(newdata))
  data.frame(Yes = pred, No = 1 - pred)
}

model_keras %>%
  predict_model(x_tbl_prueba, "raw") %>%
  as_tibble()

library(lime)

explainer <- x_tbl_entrenar %>%
  as_tibble() %>% 
  lime(model_keras, 
       bin_continuous = FALSE)
  
explanation <-  x_tbl_entrenar %>%
  as.data.frame() %>%
  head(40) %>%
  lime::explain(
    explainer    = explainer, 
    n_labels     = 1, 
    n_features   = 4,
    kernel_width = 0.5
    )

plot_explanations(explanation) +
  labs(
    title = "Importancia de cada variable",
    subtitle = "Usando 40 observaciones de prueba"
    )

corrr

https://github.com/drsimonj/corrr

library(corrr)

corrr_analysis <- x_tbl_entrenar %>%
  as_tibble() %>%
  mutate(Churn = y_vec_entrenar) %>%
  correlate() %>%
  focus(Churn) %>%
  rename(feature = rowname) %>%
  arrange(abs(Churn)) %>%
  mutate(feature = as_factor(feature))


Correlation method: 'pearson'
Missing treated using: 'pairwise.complete.obs'

corrr_analysis

over <- corrr_analysis %>%
  filter(Churn > 0)

under <- corrr_analysis %>%
  filter(Churn < 0)

corrr_analysis %>%
  ggplot(aes(x = Churn, y = fct_reorder(feature, desc(Churn)))) +
    geom_point() +
    geom_segment(aes(xend = 0, yend = feature), data = under, color = "orange") +
    geom_point(data = under, color = "orange") +
    geom_segment(aes(xend = 0, yend = feature), data = over, color = "blue") +
    geom_point(data = over, color = "blue") +
  labs(title = "Corelaciones de perdida de clientes", y = "", x = "")

NA

Mas exploracion

datos_perdimiento %>%
  group_by(Contract, Churn) %>%
  tally() %>%
  spread(Churn, n)

datos_perdimiento %>%
  group_by(InternetService, Churn) %>%
  tally() %>%
  spread(Churn, n)

Desplegar el modelo

export_savedmodel(model_keras, "tfmodel")

library(rsconnect)
deployTFModel(
  "tfmodel", 
  server = "colorado.rstudio.com", 
  account = rstudioapi::askForPassword("Enter Connect Username:")
  )

library(httr)

baked_numeric <- x_tbl_prueba %>%
  as_tibble() %>%
  head(4) %>%
  transpose() %>%
  map(as.numeric)

body <- list(instances = list(baked_numeric))

r <- POST("https://colorado.rstudio.com/rsc/content/2230/serving_default/predict", body = body, encode = "json")

jsonlite::fromJSON(content(r))$predictions[, , 1]

[1] 0.5589350 0.2727856 0.5589350 0.5589350

LS0tDQp0aXRsZTogIkFwcmVuZGl6YWplIEF1dG9tYXRpY28gY29uIFRlbnNvcmZsb3cgeSBSIg0Kb3V0cHV0OiBodG1sX25vdGVib29rDQotLS0NCg0KIyMgSW5zdGFsYXIgcGFxdWV0ZXMNCg0KYGBge3IsIGV2YWwgPSBGQUxTRX0NCnBrZ3MgPC0gYygia2VyYXMiLCAibGltZSIsICJyc2FtcGxlIiwgInJlY2lwZXMiLCAieWFyZHN0aWNrIiwgImNvcnJyIikNCmluc3RhbGwucGFja2FnZXMocGtncykNCmBgYA0KDQpgYGB7ciwgaW5jbHVkZSA9IEZBTFNFfQ0KbGlicmFyeShrZXJhcykNCmxpYnJhcnkobGltZSkNCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeShyc2FtcGxlKQ0KbGlicmFyeShyZWNpcGVzKQ0KbGlicmFyeSh5YXJkc3RpY2spDQpsaWJyYXJ5KGNvcnJyKQ0KbGlicmFyeSh0ZW5zb3JmbG93KQ0KYGBgDQoNCiMjIHRpZHl2ZXJzZQ0KDQpodHRwOi8vdGlkeXZlcnNlLm9yZy8NCg0KDQpgYGB7ciwgZWNobyA9IEZBTFNFfQ0KbGlicmFyeSh0aWR5dmVyc2UpDQoNCmlmKCFmaWxlLmV4aXN0cygiY3VzdG9tZXJfY2h1cm4uY3N2Iikpew0KICBkb3dubG9hZC5maWxlKA0KICAgICJodHRwczovL3Jhdy5naXRodWJ1c2VyY29udGVudC5jb20vcnN0dWRpby9rZXJhcy1jdXN0b21lci1jaHVybi9tYXN0ZXIvZGF0YS9XQV9Gbi1Vc2VDXy1UZWxjby1DdXN0b21lci1DaHVybi5jc3YiLA0KICAgICJjdXN0b21lcl9jaHVybi5jc3YiDQogICkgDQp9DQoNCmRhdG9zX3BlcmRpbWllbnRvIDwtIHJlYWRfY3N2KCJjdXN0b21lcl9jaHVybi5jc3YiKQ0KYGBgDQoNCmBgYHtyLCBldmFsID0gRkFMU0V9DQpnbGltcHNlKGRhdG9zX3BlcmRpbWllbnRvKQ0KYGBgDQoNCg0KIyMgcnNhbXBsZQ0KDQpodHRwczovL3RpZHltb2RlbHMuZ2l0aHViLmlvL3JzYW1wbGUvDQoNCmBgYHtyfQ0KbGlicmFyeShyc2FtcGxlKQ0KDQpzZXQuc2VlZCgxMDApDQoNCnNlcGFyYV9kYXRvcyA8LSBpbml0aWFsX3NwbGl0KA0KICBkYXRvc19wZXJkaW1pZW50bywgDQogIHByb3AgPSAwLjMpDQoNCnRibF9lbnRyZW5hciA8LSB0cmFpbmluZyhzZXBhcmFfZGF0b3MpDQp0YmxfcHJ1ZWJhICA8LSB0ZXN0aW5nKHNlcGFyYV9kYXRvcykNCmBgYA0KDQojIyByZWNpcGVzDQoNCmh0dHBzOi8vdGlkeW1vZGVscy5naXRodWIuaW8vcmVjaXBlcy8NCg0KYGBge3J9DQpsaWJyYXJ5KHJlY2lwZXMpDQoNCnJlY2V0YSA8LSB0YmxfZW50cmVuYXIgJT4lDQogIHJlY2lwZShDaHVybiB+IC4pICU+JQ0KICBzdGVwX3JtKGN1c3RvbWVySUQpICU+JQ0KICBzdGVwX25hb21pdChhbGxfb3V0Y29tZXMoKSwgYWxsX3ByZWRpY3RvcnMoKSkgJT4lDQogIHN0ZXBfZGlzY3JldGl6ZSh0ZW51cmUsIG9wdGlvbnMgPSBsaXN0KGN1dHMgPSA2KSkgJT4lDQogIHN0ZXBfbG9nKFRvdGFsQ2hhcmdlcykgJT4lDQogIHN0ZXBfbXV0YXRlKENodXJuID0gaWZlbHNlKENodXJuID09ICJZZXMiLCAxLCAwKSkgJT4lDQogIHN0ZXBfZHVtbXkoYWxsX25vbWluYWwoKSwgLWFsbF9vdXRjb21lcygpKSAlPiUNCiAgc3RlcF9jZW50ZXIoYWxsX3ByZWRpY3RvcnMoKSwgLWFsbF9vdXRjb21lcygpKSAlPiUNCiAgc3RlcF9zY2FsZShhbGxfcHJlZGljdG9ycygpLCAtYWxsX291dGNvbWVzKCkpICU+JQ0KICBwcmVwKCkNCg0Kc3VtbWFyeShyZWNldGEpDQpgYGANCg0KYGBge3J9DQpzYXZlKHJlY2V0YSwgZmlsZSA9ICIuLi9hcGxpY2FjaW9uL3JlY2V0YS5SRGF0YSIpDQpgYGANCg0KYGBge3J9DQp4X3RibF9lbnRyZW5hciA8LSByZWNldGEgJT4lIA0KICBqdWljZShhbGxfcHJlZGljdG9ycygpLCBjb21wb3NpdGlvbiA9ICJtYXRyaXgiKSANCg0KeV92ZWNfZW50cmVuYXIgPC0gcmVjZXRhICU+JSANCiAganVpY2UoYWxsX291dGNvbWVzKCkpICU+JSANCiAgcHVsbCgpDQpgYGANCg0KDQpgYGB7cn0NCmJha2VkX3Rlc3QgPC0gYmFrZShyZWNldGEsIHRibF9wcnVlYmEpDQoNCnhfdGJsX3BydWViYSA8LSBiYWtlZF90ZXN0ICU+JQ0KICBzZWxlY3QoLUNodXJuKSAlPiUNCiAgYXMubWF0cml4KCkNCg0KeV92ZWNfcHJ1ZWJhIDwtIGJha2VkX3Rlc3QgJT4lDQogIHNlbGVjdChDaHVybikgJT4lDQogIHB1bGwoKQ0KYGBgDQoNCg0KIyMgSW5zdGFsYXIgVGVuc29yZmxvdyAmIEtlcmFzDQoNCmh0dHBzOi8vdGVuc29yZmxvdy5yc3R1ZGlvLmNvbS90ZW5zb3JmbG93L2FydGljbGVzL2luc3RhbGxhdGlvbi5odG1sDQoNCmh0dHBzOi8vdGVuc29yZmxvdy5yc3R1ZGlvLmNvbS9rZXJhcy8jaW5zdGFsbGF0aW9uDQoNCmBgYHtyLCBldmFsID0gRkFMU0UgfQ0KbGlicmFyeSh0ZW5zb3JmbG93KQ0KbGlicmFyeShrZXJhcykNCg0KI2luc3RhbGxfdGVuc29yZmxvdygpDQojaW5zdGFsbF9rZXJhcygpDQpgYGANCg0KDQojIyMgQ3JlYXIgdW5hIHJlZCBuZXVyYWwNCg0KYGBge3J9DQptb2RlbF9rZXJhcyA8LSBrZXJhc19tb2RlbF9zZXF1ZW50aWFsKCkgJT4lDQogIGxheWVyX2RlbnNlKA0KICAgIHVuaXRzID0gMTYsIA0KICAgIGtlcm5lbF9pbml0aWFsaXplciA9ICJ1bmlmb3JtIiwgDQogICAgYWN0aXZhdGlvbiA9ICJyZWx1IiwgDQogICAgaW5wdXRfc2hhcGUgPSBuY29sKHhfdGJsX2VudHJlbmFyKSkgJT4lIA0KICBsYXllcl9kcm9wb3V0KHJhdGUgPSAwLjEpICU+JQ0KICBsYXllcl9kZW5zZSgNCiAgICB1bml0cyA9IDE2LCANCiAgICBrZXJuZWxfaW5pdGlhbGl6ZXIgPSAidW5pZm9ybSIsIA0KICAgIGFjdGl2YXRpb24gPSAicmVsdSIpICU+JSANCiAgbGF5ZXJfZHJvcG91dChyYXRlID0gMC4xKSAlPiUNCiAgbGF5ZXJfZGVuc2UoDQogICAgdW5pdHMgPSAxLCANCiAgICBrZXJuZWxfaW5pdGlhbGl6ZXIgPSAidW5pZm9ybSIsIA0KICAgIGFjdGl2YXRpb24gPSAic2lnbW9pZCIpICU+JSANCiAgY29tcGlsZSgNCiAgICBvcHRpbWl6ZXIgPSAnYWRhbScsDQogICAgbG9zcyA9ICdiaW5hcnlfY3Jvc3NlbnRyb3B5JywNCiAgICBtZXRyaWNzID0gYygnYWNjdXJhY3knKQ0KICApDQoNCm1vZGVsX2tlcmFzDQpgYGANCg0KIyMjIENvcnJlciBlbCBtb2RlbG8NCg0KYGBge3J9DQpoaXN0b3J5IDwtIGZpdCgNCiAgb2JqZWN0ID0gbW9kZWxfa2VyYXMsIA0KICB4ID0geF90YmxfZW50cmVuYXIsIA0KICB5ID0geV92ZWNfZW50cmVuYXIsDQogIGJhdGNoX3NpemUgPSA1MCwgDQogIGVwb2NocyA9IDM1LA0KICB2YWxpZGF0aW9uX3NwbGl0ID0gMC4zMCwNCiAgdmVyYm9zZSA9IDANCikNCg0KcHJpbnQoaGlzdG9yeSkNCmBgYA0KDQojIyMgVmVyIGxvcyByZXN1bHRhZG9zDQoNCmBgYHtyfQ0KdGhlbWVfc2V0KHRoZW1lX2J3KCkpDQoNCnBsb3QoaGlzdG9yeSkgDQpgYGANCg0KYGBge3J9DQp5aGF0X2tlcmFzX2NsYXNzX3ZlYyA8LSBtb2RlbF9rZXJhcyAlPiUNCiAgcHJlZGljdF9jbGFzc2VzKHhfdGJsX3BydWViYSkgJT4lDQogIGFzLmZhY3RvcigpICU+JQ0KICBmY3RfcmVjb2RlKHllcyA9ICIxIiwgbm8gPSAiMCIpDQoNCnloYXRfa2VyYXNfcHJvYl92ZWMgIDwtIG1vZGVsX2tlcmFzICU+JQ0KICBwcmVkaWN0X3Byb2JhKHhfdGJsX3BydWViYSkgJT4lDQogIGFzLnZlY3RvcigpDQoNCnRlc3RfdHJ1dGggPC0geV92ZWNfcHJ1ZWJhICU+JSANCiAgYXMuZmFjdG9yKCkgJT4lIA0KICBmY3RfcmVjb2RlKHllcyA9ICIxIiwgbm8gPSAiMCIpDQoNCmVzdGltYXRlc19rZXJhc190YmwgPC0gdGliYmxlKA0KICB0cnV0aCAgICAgID0gdGVzdF90cnV0aCwNCiAgZXN0aW1hdGUgICA9IHloYXRfa2VyYXNfY2xhc3NfdmVjLA0KICBjbGFzc19wcm9iID0geWhhdF9rZXJhc19wcm9iX3ZlYw0KKQ0KDQplc3RpbWF0ZXNfa2VyYXNfdGJsDQpgYGANCg0KDQojIyB5YXJkc3RpY2sNCg0KaHR0cHM6Ly90aWR5bW9kZWxzLmdpdGh1Yi5pby95YXJkc3RpY2svDQoNCmB5YXJkc3RpY2tgIGlzIGEgcGFja2FnZSB0byBlc3RpbWF0ZSBob3cgd2VsbCBtb2RlbHMgYXJlIHdvcmtpbmcgdXNpbmcgdGlkeSBkYXRhIHByaW5jaXBhbHMuDQoNCmBgYHtyfQ0KbGlicmFyeSh5YXJkc3RpY2spDQoNCm9wdGlvbnMoeWFyZHN0aWNrLmV2ZW50X2ZpcnN0ID0gRkFMU0UpDQoNCmVzdGltYXRlc19rZXJhc190YmwgJT4lIA0KICBjb25mX21hdCh0cnV0aCwgZXN0aW1hdGUpDQoNCmVzdGltYXRlc19rZXJhc190YmwgJT4lIA0KICBtZXRyaWNzKHRydXRoLCBlc3RpbWF0ZSkNCg0KZXN0aW1hdGVzX2tlcmFzX3RibCAlPiUgDQogIHJvY19hdWModHJ1dGgsIGNsYXNzX3Byb2IpDQoNCmVzdGltYXRlc19rZXJhc190YmwgJT4lDQogIHByZWNpc2lvbih0cnV0aCwgZXN0aW1hdGUpICU+JQ0KICBiaW5kX3Jvd3MoDQogICAgZXN0aW1hdGVzX2tlcmFzX3RibCAlPiUgDQogICAgICByZWNhbGwodHJ1dGgsIGVzdGltYXRlKSANCiAgKSANCg0KZXN0aW1hdGVzX2tlcmFzX3RibCAlPiUgDQogIGZfbWVhcyh0cnV0aCwgZXN0aW1hdGUsIGJldGEgPSAxKQ0KYGBgDQoNCiMjIGxpbWUNCg0KaHR0cHM6Ly9naXRodWIuY29tL3Rob21hc3A4NS9saW1lDQoNCmBgYHtyfQ0KbGlicmFyeShsaW1lKQ0KDQptb2RlbF90eXBlLmtlcmFzLmVuZ2luZS5zZXF1ZW50aWFsLlNlcXVlbnRpYWwgPC0gZnVuY3Rpb24oeCwgLi4uKSB7DQogICJjbGFzc2lmaWNhdGlvbiINCn0NCg0KcHJlZGljdF9tb2RlbC5rZXJhcy5lbmdpbmUuc2VxdWVudGlhbC5TZXF1ZW50aWFsIDwtIGZ1bmN0aW9uKHgsIG5ld2RhdGEsIHR5cGUsIC4uLikgew0KICBwcmVkIDwtIHByZWRpY3RfcHJvYmEob2JqZWN0ID0geCwgeCA9IGFzLm1hdHJpeChuZXdkYXRhKSkNCiAgZGF0YS5mcmFtZShZZXMgPSBwcmVkLCBObyA9IDEgLSBwcmVkKQ0KfQ0KYGBgDQoNCg0KYGBge3J9DQptb2RlbF9rZXJhcyAlPiUNCiAgcHJlZGljdF9tb2RlbCh4X3RibF9wcnVlYmEsICJyYXciKSAlPiUNCiAgYXNfdGliYmxlKCkNCmBgYA0KDQpgYGB7cn0NCmxpYnJhcnkobGltZSkNCg0KZXhwbGFpbmVyIDwtIHhfdGJsX2VudHJlbmFyICU+JQ0KICBhc190aWJibGUoKSAlPiUgDQogIGxpbWUobW9kZWxfa2VyYXMsIA0KICAgICAgIGJpbl9jb250aW51b3VzID0gRkFMU0UpDQogIA0KZXhwbGFuYXRpb24gPC0gIHhfdGJsX2VudHJlbmFyICU+JQ0KICBhcy5kYXRhLmZyYW1lKCkgJT4lDQogIGhlYWQoNDApICU+JQ0KICBsaW1lOjpleHBsYWluKA0KICAgIGV4cGxhaW5lciAgICA9IGV4cGxhaW5lciwgDQogICAgbl9sYWJlbHMgICAgID0gMSwgDQogICAgbl9mZWF0dXJlcyAgID0gNCwNCiAgICBrZXJuZWxfd2lkdGggPSAwLjUNCiAgICApDQpgYGANCg0KDQpgYGB7ciwgZmlnLndpZHRoID0gMTB9DQpwbG90X2V4cGxhbmF0aW9ucyhleHBsYW5hdGlvbikgKw0KICBsYWJzKA0KICAgIHRpdGxlID0gIkltcG9ydGFuY2lhIGRlIGNhZGEgdmFyaWFibGUiLA0KICAgIHN1YnRpdGxlID0gIlVzYW5kbyA0MCBvYnNlcnZhY2lvbmVzIGRlIHBydWViYSINCiAgICApDQpgYGANCg0KDQojIyBjb3Jycg0KDQpodHRwczovL2dpdGh1Yi5jb20vZHJzaW1vbmovY29ycnINCg0KYGBge3J9DQpsaWJyYXJ5KGNvcnJyKQ0KDQpjb3Jycl9hbmFseXNpcyA8LSB4X3RibF9lbnRyZW5hciAlPiUNCiAgYXNfdGliYmxlKCkgJT4lDQogIG11dGF0ZShDaHVybiA9IHlfdmVjX2VudHJlbmFyKSAlPiUNCiAgY29ycmVsYXRlKCkgJT4lDQogIGZvY3VzKENodXJuKSAlPiUNCiAgcmVuYW1lKGZlYXR1cmUgPSByb3duYW1lKSAlPiUNCiAgYXJyYW5nZShhYnMoQ2h1cm4pKSAlPiUNCiAgbXV0YXRlKGZlYXR1cmUgPSBhc19mYWN0b3IoZmVhdHVyZSkpIA0KDQpjb3Jycl9hbmFseXNpcw0KYGBgDQoNCmBgYHtyLCBmaWcuaGVpZ2h0ID0gNywgZmlnLndpZHRoID0gN30NCm92ZXIgPC0gY29ycnJfYW5hbHlzaXMgJT4lDQogIGZpbHRlcihDaHVybiA+IDApDQoNCnVuZGVyIDwtIGNvcnJyX2FuYWx5c2lzICU+JQ0KICBmaWx0ZXIoQ2h1cm4gPCAwKQ0KDQpjb3Jycl9hbmFseXNpcyAlPiUNCiAgZ2dwbG90KGFlcyh4ID0gQ2h1cm4sIHkgPSBmY3RfcmVvcmRlcihmZWF0dXJlLCBkZXNjKENodXJuKSkpKSArDQogICAgZ2VvbV9wb2ludCgpICsNCiAgICBnZW9tX3NlZ21lbnQoYWVzKHhlbmQgPSAwLCB5ZW5kID0gZmVhdHVyZSksIGRhdGEgPSB1bmRlciwgY29sb3IgPSAib3JhbmdlIikgKw0KICAgIGdlb21fcG9pbnQoZGF0YSA9IHVuZGVyLCBjb2xvciA9ICJvcmFuZ2UiKSArDQogICAgZ2VvbV9zZWdtZW50KGFlcyh4ZW5kID0gMCwgeWVuZCA9IGZlYXR1cmUpLCBkYXRhID0gb3ZlciwgY29sb3IgPSAiYmx1ZSIpICsNCiAgICBnZW9tX3BvaW50KGRhdGEgPSBvdmVyLCBjb2xvciA9ICJibHVlIikgKw0KICBsYWJzKHRpdGxlID0gIkNvcmVsYWNpb25lcyBkZSBwZXJkaWRhIGRlIGNsaWVudGVzIiwgeSA9ICIiLCB4ID0gIiIpDQogIA0KYGBgDQoNCiMjIE1hcyBleHBsb3JhY2lvbg0KDQpgYGB7cn0NCmRhdG9zX3BlcmRpbWllbnRvICU+JQ0KICBncm91cF9ieShDb250cmFjdCwgQ2h1cm4pICU+JQ0KICB0YWxseSgpICU+JQ0KICBzcHJlYWQoQ2h1cm4sIG4pDQpgYGANCg0KYGBge3J9DQpkYXRvc19wZXJkaW1pZW50byAlPiUNCiAgZ3JvdXBfYnkoSW50ZXJuZXRTZXJ2aWNlLCBDaHVybikgJT4lDQogIHRhbGx5KCkgJT4lDQogIHNwcmVhZChDaHVybiwgbikNCmBgYA0KDQojIyBEZXNwbGVnYXIgZWwgbW9kZWxvDQoNCg0KYGBge3IsIGV2YWwgPSBGQUxTRX0NCmV4cG9ydF9zYXZlZG1vZGVsKG1vZGVsX2tlcmFzLCAidGZtb2RlbCIpDQpgYGANCg0KYGBge3IsZXZhbCA9IEZBTFNFfQ0KbGlicmFyeShyc2Nvbm5lY3QpDQpkZXBsb3lURk1vZGVsKA0KICAidGZtb2RlbCIsIA0KICBzZXJ2ZXIgPSAiY29sb3JhZG8ucnN0dWRpby5jb20iLCANCiAgYWNjb3VudCA9IHJzdHVkaW9hcGk6OmFza0ZvclBhc3N3b3JkKCJFbnRlciBDb25uZWN0IFVzZXJuYW1lOiIpDQogICkNCmBgYA0KDQpgYGB7cn0NCmxpYnJhcnkoaHR0cikNCg0KYmFrZWRfbnVtZXJpYyA8LSB4X3RibF9wcnVlYmEgJT4lDQogIGFzX3RpYmJsZSgpICU+JQ0KICBoZWFkKDQpICU+JQ0KICB0cmFuc3Bvc2UoKSAlPiUNCiAgbWFwKGFzLm51bWVyaWMpDQoNCmJvZHkgPC0gbGlzdChpbnN0YW5jZXMgPSBsaXN0KGJha2VkX251bWVyaWMpKQ0KDQpyIDwtIFBPU1QoImh0dHBzOi8vY29sb3JhZG8ucnN0dWRpby5jb20vcnNjL2NvbnRlbnQvMjIzMC9zZXJ2aW5nX2RlZmF1bHQvcHJlZGljdCIsIGJvZHkgPSBib2R5LCBlbmNvZGUgPSAianNvbiIpDQoNCmpzb25saXRlOjpmcm9tSlNPTihjb250ZW50KHIpKSRwcmVkaWN0aW9uc1ssICwgMV0NCmBgYA0K