## ----setup, include = FALSE---------------------------------------------------
library(keras)
knitr::opts_chunk$set(comment = NA, eval = FALSE)

## -----------------------------------------------------------------------------
#  library(tensorflow)
#  library(keras)
#  
#  layer_linear <- Layer(
#    classname = "Linear",
#    initialize = function(units, input_dim) {
#      super()$`__init__`()
#      w_init <- tf$random_normal_initializer()
#      self$w <- tf$Variable(
#        initial_value = w_init(shape = shape(input_dim, units),
#                               dtype = tf$float32)
#        )
#      b_init <- tf$zeros_initializer()
#      self$b <- tf$Variable(
#        initial_value = b_init(shape = shape(units),
#                               dtype = tf$float32)
#      )
#    },
#    call = function(inputs, ...) {
#      tf$matmul(inputs, self$w) + self$b
#    }
#  )
#  
#  x <- tf$ones(shape = list(2,2))
#  layer <- layer_linear(units = 4, input_dim = 2)
#  y <- layer(x)
#  y

## -----------------------------------------------------------------------------
#  get_weights(layer)

## -----------------------------------------------------------------------------
#  layer_linear <- Layer(
#    classname = "Linear",
#    initialize = function(units, input_dim) {
#      super()$`__init__`()
#      self$w <- self$add_weight(
#        shape = shape(input_dim, units),
#        initializer = "random_normal",
#        trainable = TRUE
#      )
#      self$b <- self$add_weight(
#        shape = shape(units),
#        initializer = "zeros",
#        trainable = TRUE
#      )
#    },
#    call = function(inputs, ...) {
#      tf$matmul(inputs, self$w) + self$b
#    }
#  )

## -----------------------------------------------------------------------------
#  layer_compute_sum <- Layer(
#    classname = "ComputeSum",
#    initialize = function(input_dim) {
#      super()$`__init__`()
#      self$total <- tf$Variable(
#        initial_value = tf$zeros(shape(input_dim)),
#        trainable = FALSE
#      )
#    },
#    call = function(inputs, ...) {
#      self$total$assign_add(tf$reduce_sum(inputs, axis = 0L))
#      self$total
#    }
#  )
#  
#  x <- tf$ones(shape(2,2))
#  mysum <- layer_compute_sum(input_dim = 2)
#  print(mysum(x))
#  print(mysum(x))

## -----------------------------------------------------------------------------
#  get_weights(mysum)
#  mysum$non_trainable_weights

## -----------------------------------------------------------------------------
#  layer_linear <- Layer(
#    classname = "Linear",
#    initialize = function(units, input_dim) {
#      super()$`__init__`()
#      self$w <- self$add_weight(
#        shape = shape(input_dim, units),
#        initializer = "random_normal",
#        trainable = TRUE
#      )
#      self$b <- self$add_weight(
#        shape = shape(units),
#        initializer = "zeros",
#        trainable = TRUE
#      )
#    },
#    call = function(inputs, ...) {
#      tf$matmul(inputs, self$w) + self$b
#    }
#  )

## -----------------------------------------------------------------------------
#  layer_linear <- Layer(
#    classname = "Linear",
#    initialize = function(units) {
#      super()$`__init__`()
#      self$units <- units
#    },
#    build = function(input_shape) {
#      self$w <- self$add_weight(
#        shape = shape(input_shape[2], self$units),
#        initializer = "random_normal",
#        trainable = TRUE
#      )
#      self$b <- self$add_weight(
#        shape = shape(self$units),
#        initializer = "zeros",
#        trainable = TRUE
#      )
#    },
#    call = function(inputs, ...) {
#      tf$matmul(inputs, self$w) + self$b
#    }
#  )

## -----------------------------------------------------------------------------
#  layer <- layer_linear(units = 32)
#  x <- tf$ones(shape = list(2,2))
#  layer(x)

## -----------------------------------------------------------------------------
#  # Let's assume we are reusing the Linear class
#  # with a `build` method that we defined above.
#  layer_mlp_block <- Layer(
#    classname = "MLPBlock",
#    initialize = function() {
#      super()$`__init__`()
#      self$linear_1 <- layer_linear(units = 32)
#      self$linear_2 <- layer_linear(units = 32)
#      self$linear_3 <- layer_linear(units = 1)
#    },
#    call = function(inputs, ...) {
#      inputs %>%
#        self$linear_1() %>%
#        tf$nn$relu() %>%
#        self$linear_2() %>%
#        tf$nn$relu() %>%
#        self$linear_3()
#    }
#  )
#  
#  mlp <- layer_mlp_block()
#  
#  y <- mlp(tf$ones(shape(3, 64)))  # The first call to the `mlp` will create the weights
#  length(mlp$weights)
#  length(mlp$trainable_weights)

## -----------------------------------------------------------------------------
#  # A layer that creates an activity regularization loss
#  layer_activity_reg <- Layer(
#    classname = "ActivityRegularizationLayer",
#    initialize = function(rate = 1e-2) {
#      super()$`__init__`()
#      self$rate <- rate
#    },
#    call = function(inputs) {
#      self$add_loss(self$rate * tf$reduce_sum(inputs))
#      inputs
#    }
#  )

## -----------------------------------------------------------------------------
#  layer_outer <- Layer(
#    classname = "OuterLayer",
#    initialize = function() {
#      super()$`__init__`()
#      self$dense <- layer_dense(
#        units = 32,
#        kernel_regularizer = regularizer_l2(1e-3)
#      )
#    },
#    call = function(inputs) {
#      self$dense(inputs)
#    }
#  )
#  
#  layer <- layer_outer()
#  x <- layer(tf$zeros(shape(1,1)))
#  
#  # This is `1e-3 * sum(layer.dense.kernel ** 2)`,
#  # created by the `kernel_regularizer` above.
#  layer$losses

## -----------------------------------------------------------------------------
#  layer_linear <- Layer(
#    classname = "Linear",
#    initialize = function(units, ...) {
#      super()$`__init__`(...)
#      self$units <- units
#    },
#    build = function(input_shape) {
#      self$w <- self$add_weight(
#        shape = shape(input_shape[2], self$units),
#        initializer = "random_normal",
#        trainable = TRUE
#      )
#      self$b <- self$add_weight(
#        shape = shape(self$units),
#        initializer = "zeros",
#        trainable = TRUE
#      )
#    },
#    call = function(inputs, ...) {
#      tf$matmul(inputs, self$w) + self$b
#    },
#    get_config = function() {
#      list(
#        units = self$units
#      )
#    }
#  )
#  
#  layer <- layer_linear(units = 64)
#  config <- get_config(layer)
#  new_layer <- from_config(config)

## -----------------------------------------------------------------------------
#  layer_custom_dropout <- Layer(
#    classname =  "CustomDropout",
#    initialize = function(rate, ...) {
#      super()$`__init__`(...)
#      self$rate <- rate
#    },
#    call = function(inputs, training = NULL) {
#      if (!is.null(inputs) && training) {
#        inputs <- tf$nn$dropout(inputs, rate = self$rate)
#      }
#      inputs
#    }
#  )