Nonlinear computation in deep linear networks

1

x = tf.placeholder(dtype=tf.float32, shape=[batch_size,784])

2

y = tf.placeholder(dtype=tf.float32, shape=[batch_size,10])

3

4

w1 = tf.Variable(np.random.normal(scale=np.sqrt(2./784),size=[784,512]).astype(np.float32))

5

b1 = tf.Variable(np.zeros(512,dtype=np.float32))

6

w2 = tf.Variable(np.random.normal(scale=np.sqrt(2./512),size=[512,512]).astype(np.float32))

7

b2 = tf.Variable(np.zeros(512,dtype=np.float32))

8

w3 = tf.Variable(np.random.normal(scale=np.sqrt(2./512),size=[512,10]).astype(np.float32))

9

b3 = tf.Variable(np.zeros(10,dtype=np.float32))

10

11

params = [w1,b1,w2,b2,w3,b3]

12

nr_params = sum([np.prod(p.get_shape().as_list()) for p in params])

13

scaling = 2**125

14

15

def get_logits(par):

16

h1 = tf.nn.bias_add(tf.matmul(x , par[0]), par[1]) / scaling

17

h2 = tf.nn.bias_add(tf.matmul(h1, par[2]) , par[3] / scaling)

18

o = tf.nn.bias_add(tf.matmul(h2, par[4]), par[5]/ scaling)*scaling

19

return o