Tensorflow Experience

# 使用分批数据，处理所有的不同数据内容
d = 1024  # Tox21 特征向量维度数
with tf.name_scope("placeholdes"):
    x = tf.placeholder(tf.float32, (None, d))
    y = tf.placeholder(tf.float32, (None, ))

# 定义一个隐藏层
with tf.name_scope("hidden-layer"):
    W = tf.Variable(tf.random_normal((d, n_hidden)))
    b = tf.Variable(tf.random_normal((n_hidden, )))
    # 使用xW而不是Wx，为了方便处理多批次的数据
    # 使用ReLU的非线性整流结果
    # max(0, w^Tx + b)
    x_hidden = tf.nn.relu(tf.matmul(x, W) + b)

# 使用分批数据，处理所有的不同数据内容
d = 1024  # 特征向量维度数
with tf.name_scope("placeholdes"):
    x = tf.placeholder(tf.float32, (None, d))
    y = tf.placeholder(tf.float32, (None, ))

# 定义一个隐藏层
with tf.name_scope("hidden-layer"):
    W = tf.Variable(tf.random_normal((d, n_hidden)))
    b = tf.Variable(tf.random_normal((n_hidden, )))
    # 使用xW而不是Wx，为了方便处理多批次的数据
    # 使用ReLU的非线性整流结果
    # max(0, w^Tx + b)
    x_hidden = tf.nn.relu(tf.matmul(x, W) + b)

# 定义输出层
with tf.name_scope("output"):
    W = tf.Variable(tf.random_normal((n_hidden, 1)))
    b = tf.Variable(tf.random_normal((1, )))
    y_logit = tf.matmul(x_hidden, W) + b
    y_one_prob = tf.sigmoid(y_logit)
    y_pred = tf.round(y_one_prob)

# 定义损失函数
with tf.name_scope("loss"):
    y_expand = tf.expand_dims(y, 1)
    # 交叉熵差异损失函数
    entropy = tf.nn.sigmoid_cross_entropy_with_logits(
        logits=y_logit, labels=y_expand)
    l = tf.reduce_sum(entropy)

# 定义下降函数
with tf.name_scope("optim"):
    train_op = tf.train.AdamOptimizer(learning_rate).minimize(l)

with tf.name_scope("summaries"):
    tf.summary.scalar("loss", l)
    merged = tf.summary.merge_all()

train_writer = tf.summary.FileWriter('logs/train', tf.get_default_graph())

with tf.name_scope("placeholdes"):
    x = tf.placeholder(tf.float32, (None, d))
    y = tf.placeholder(tf.float32, (None, ))
    keep_prob = tf.placeholder(tf.float32)

# 定义一个隐藏层
with tf.name_scope("hidden-layer"):
    W = tf.Variable(tf.random_normal((d, n_hidden)))
    b = tf.Variable(tf.random_normal((n_hidden, )))
    # 使用xW而不是Wx，为了方便处理多批次的数据
    # 使用ReLU的非线性整流结果
    # max(0, w^Tx + b)
    x_hidden = tf.nn.relu(tf.matmul(x, W) + b)
    # add dropout
    x_hidden = tf.nn.dropout(x_hidden, keep_prob)

# 喂入批量数据
step = 0

N = train_X.shape[0]

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())

    for epoch in range(n_epochs):
        pos = 0
        while pos < N:
            batch_x = train_X[pos:pos + batch_size]
            batch_y = train_y[pos:pos + batch_size]
            feed_dict = {x: batch_x, y: batch_y, keep_prob: dropout_prob}
            _, summary, loss = sess.run([train_op, merged, l],
                                        feed_dict=feed_dict)
            print("epoch %d, step %d, loss: %f" % (epoch, step, loss))

            train_writer.add_summary(summary, step)

            step += 1
            pos += batch_size

    # 得到预测结果
    train_y_pred = sess.run(y_pred, feed_dict={x: train_X, keep_prob: 1.0})
    valid_y_pred = sess.run(y_pred, feed_dict={x: valid_X, keep_prob: 1.0})
    test_y_pred = sess.run(y_pred, feed_dict={x: test_X, keep_prob: 1.0})

# 定义评价函数，由于0标签数量比1标签数量多的多，我们需要给1标签数量加权重
train_weighted_score = accuracy_score(
    train_y, train_y_pred, sample_weight=train_w)
print("Train Weighted Classfiication Accuracy: %f" % train_weighted_score)
valid_weighted_score = accuracy_score(
    valid_y, valid_y_pred, sample_weight=valid_w)
print("Valid Weighted Classfication Accuracy: %f" % valid_weighted_score)
test_weighted_score = accuracy_score(test_y, test_y_pred, sample_weight=test_w)
print("Test Weighted Classfication Accuracy: %f" % test_weighted_score)

tensorboard  --logdir logs/train

%tensorflow_version 1.x
import tensorflow as tf
import numpy as np
import pandas as pd

tf.logging.set_verbosity(tf.logging.ERROR)
pd.options.display.max_rows = 10
pd.options.display.float_format = '{:.1f}'.format

# 载入 mnist 训练数据, 用 pandas 的 dataframe 读数据
mnist_dataframe = pd.read_csv(
  "https://download.mlcc.google.cn/mledu-datasets/mnist_train_small.csv",
  sep=",",
  header=None)

# 只使用前10000个数据做实验
mnist_dataframe = mnist_dataframe.head(10000)
mnist_dataframe = mnist_dataframe.reindex(np.random.permutation(mnist_dataframe.index))
# 展示一下数据
mnist_dataframe.head()

def parse_labels_and_features(dataset):
  """从数据中提取标签和特征

  Args:
    dataset: Dataframe, 第一列是标签，余下的列为单色的像素数据

  Returns:
    A `tuple` `(labels, features)`:
      labels: A Pandas `Series`.
      features: A Pandas `DataFrame`.
  """
  labels = dataset[0]
  # DataFrame.loc index ranges are inclusive at both ends.
  features = dataset.loc[:,1:784]
  # 原值是 0~255，归一化到 0~1.
  features = features / 255

  return labels, features

# 将数据的前7500个做为训练集，将标签和数据拆分
training_targets, training_examples = parse_labels_and_features(mnist_dataframe[:7500])

# 拆出验证集的数据，用数据的后2500个
validation_targets, validation_examples = parse_labels_and_features(mnist_dataframe[7500:10000])

# 显示一个随机的数据
from matplotlib import pyplot as plt
from IPython import display

rand_example = np.random.choice(training_examples.index)
_, ax = plt.subplots()
ax.matshow(training_examples.loc[rand_example].values.reshape(28, 28))
ax.set_title("Label: %i" % training_targets.loc[rand_example])
ax.grid(False)

# 载入 mnist 测试数据
test_dataframe = pd.read_csv(
  "https://download.mlcc.google.cn/mledu-datasets/mnist_test.csv",
  sep=",",
  header=None)

# 只使用前10000个数据做测试
test_dataframe = test_dataframe.head(5000)
test_dataframe = test_dataframe.reindex(np.random.permutation(test_dataframe.index))

# 提取测试用的特征和标签
testing_targets, testing_examples = parse_labels_and_features(test_dataframe)

# 参数设定
IMAGE_SIZE = 28
NUM_CHANNELS = 1
PIXEL_DEPTH = 255
NUM_LABELS = 10
SEED = 66478  # Set to None for random seed.
BATCH_SIZE = 64 # 训练集的每批次数据量大小
NUM_EPOCHS = 10
EVAL_BATCH_SIZE = 64 # 验证集和测试集的每批次数据量大小
EVAL_FREQUENCY = 100  #用验证集作评估的频率
TRAIN_SIZE = training_targets.shape[0]

# 定义 train 用的 placeholder 来存放 input 数据
train_data_node = tf.placeholder(
    tf.float32,
    shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
train_labels_node = tf.placeholder(tf.int64, shape=(BATCH_SIZE,))

# 评估用的 placeholder 来存放数据
eval_data = tf.placeholder(
    tf.float32,
    shape=(EVAL_BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))

# 定义网络的权重，2层卷积层，中有2池化层
# 第一个卷积层的权重, 5x5 的大小, depth 为 32.
conv1_weights = tf.Variable(
    tf.truncated_normal([5, 5, NUM_CHANNELS, 32],  
                        stddev=0.1,
                        seed=SEED, dtype=tf.float32))
# 加一个 bais，下同
conv1_biases = tf.Variable(tf.zeros([32], dtype=tf.float32))
# 第二个卷积层，5x5x64
conv2_weights = tf.Variable(tf.truncated_normal(
    [5, 5, 32, 64], stddev=0.1,
    seed=SEED, dtype=tf.float32))
conv2_biases = tf.Variable(tf.constant(0.1, shape=[64],
                           dtype=tf.float32))

# 定义全连接网络层，由于经过了两个 2x2 的池化层，
# 输入的特征为 (IMAGE_SIZE // 4 )^2 * 64
# 全连接网络为 7*7*64 => 512 => 10
# 第一层, depth 512.
fc1_weights = tf.Variable(
    tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4 * 64, 512],
                        stddev=0.1,
                        seed=SEED,
                        dtype=tf.float32))
fc1_biases = tf.Variable(tf.constant(0.1, shape=[512],
                         dtype=tf.float32))
# 第二层，为输出层，depth 为 10
fc2_weights = tf.Variable(tf.truncated_normal([512, NUM_LABELS],
                                              stddev=0.1,
                                              seed=SEED,
                                              dtype=tf.float32))
fc2_biases = tf.Variable(tf.constant(
    0.1, shape=[NUM_LABELS], dtype=tf.float32))

# 定义网络
def model(data, train=False):
  """The Model definition."""
  # 4维的strides用以匹配数据的格式 [image index, y, x, depth]
  # 每一层的 padding 为 SAME，即输入和输出的大小补全为一样
  ###
  # 第一层
  conv = tf.nn.conv2d(data,
                      conv1_weights,
                      strides=[1, 1, 1, 1],
                      padding='SAME')
  # 使用 relu 做激活函数
  relu = tf.nn.relu(tf.nn.bias_add(conv, conv1_biases))
  # 最大池化层，窗口为2，步长为2
  pool = tf.nn.max_pool(relu,
                        ksize=[1, 2, 2, 1],
                        strides=[1, 2, 2, 1],
                        padding='SAME')
  ###
  # 第二层
  conv = tf.nn.conv2d(pool,
                      conv2_weights,
                      strides=[1, 1, 1, 1],
                      padding='SAME')
  relu = tf.nn.relu(tf.nn.bias_add(conv, conv2_biases))
  pool = tf.nn.max_pool(relu,
                        ksize=[1, 2, 2, 1],
                        strides=[1, 2, 2, 1],
                        padding='SAME')

  # 将结果 reshape 后输入到全连接网络, [image index, x*y*64]
  pool_shape = pool.get_shape().as_list()
  reshape = tf.reshape(
      pool,
      [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])

  # 一个全连接层
  hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)

  # 在训练时加入一个50%的dropout，防止过拟合
  if train:
    hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)
  return tf.matmul(hidden, fc2_weights) + fc2_biases

# 训练计算，使用 logits + cross_entropy 做损失计算
logits = model(train_data_node, True)
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
    labels=train_labels_node, logits=logits))

# 使用 L2 regularization 正则化全连接网络参数
regularizers = (tf.nn.l2_loss(fc1_weights)
                + tf.nn.l2_loss(fc1_biases)
                + tf.nn.l2_loss(fc2_weights)
                + tf.nn.l2_loss(fc2_biases))

# Add the regularization term to the loss.
loss += 5e-4 * regularizers

# 优化器：设一个变量，每批数据会增加，以控制训练速度
batch = tf.Variable(0, dtype=tf.float32)

# 每个 epoch 后降低训练速度
learning_rate = tf.train.exponential_decay(
    0.01,                # Base learning rate.
    batch * BATCH_SIZE,  # Current index into the dataset.
    TRAIN_SIZE,          # Decay step.
    0.95,                # Decay rate.
    staircase=True)

# 使用 MomentumOptimizer 做为优化器
optimizer = tf.train.MomentumOptimizer(learning_rate,
                                       0.9).minimize(loss,
                                                     global_step=batch)

# 计算预测结果
train_prediction = tf.nn.softmax(logits)

# 计算余下的结果用于 validation 和 test
eval_prediction = tf.nn.softmax(model(eval_data))

def error_rate(predictions, labels):
  """返回预测结果的误差率"""
  return 100.0 - (
      100.0 *
      np.sum(np.argmax(predictions, 1) == labels) /
      predictions.shape[0])

def eval_in_batches(data, sess):
  """分批计算网络，减少内存的使用"""
  size = data.shape[0]
  predictions = np.ndarray(shape=(size, NUM_LABELS),
                              dtype=np.float32)
  for begin in xrange(0, size, EVAL_BATCH_SIZE):
    end = begin + EVAL_BATCH_SIZE
    if end <= size:
      # 取一个 EVAL_BATCH_SIZE 的数据量，放入网络评估
      predictions[begin:end, :] = sess.run(
          eval_prediction,
          feed_dict={eval_data: data[begin:end, ...]})
    else:
      # 最后一批数据，放入余下的所有数据
      batch_predictions = sess.run(
          eval_prediction,
          feed_dict={eval_data: data[-EVAL_BATCH_SIZE:, ...]})
      predictions[begin:, :] = batch_predictions[begin - size:, :]
  return predictions

from six.moves import xrange

# 把 dataframe 转成 numpy array
train_data = training_examples.to_numpy(np.float32).reshape((-1, 28, 28, 1))
train_labels = training_targets.to_numpy(np.float32)
validation_data = validation_examples.to_numpy(np.float32).reshape((-1, 28, 28, 1))
validation_labels = validation_targets.to_numpy(np.float32)
test_data = testing_examples.to_numpy(np.float32).reshape((-1, 28, 28, 1))
test_labels = testing_targets.to_numpy(np.float32)

# 开启 TF session，运行训练
with tf.Session() as sess:
  # 初始化 session
  tf.global_variables_initializer().run()

  # 执行多步循环
  for step in xrange(int(NUM_EPOCHS * TRAIN_SIZE) // BATCH_SIZE):
    # 计算当前 batch 的位置，进行运算
    offset = (step * BATCH_SIZE) % (TRAIN_SIZE - BATCH_SIZE)
    batch_data = train_data[offset:(offset + BATCH_SIZE), ...]
    batch_labels = train_labels[offset:(offset + BATCH_SIZE)]

    # 将这一批的数据进行训练
    feed_dict = {train_data_node: batch_data,
                 train_labels_node: batch_labels}
    sess.run(optimizer, feed_dict=feed_dict)

    # 每次训练到一定的程度，用验证集做一次评估，并打印一些信息
    if step % EVAL_FREQUENCY == 0:
      # 计算一下当前的具体 loss 值，指示现在的训练进度
      l, lr, predictions = sess.run([loss, learning_rate,
                                     train_prediction],
                                    feed_dict=feed_dict)

      print('Step %d (epoch %.2f)' %
            (step, float(step) * BATCH_SIZE / TRAIN_SIZE))
      print('Minibatch loss: %.3f, learning rate: %.6f' % (l, lr))
      print('Minibatch error: %.1f%%'
            % error_rate(predictions, batch_labels))
      # 计算一下验证集的错误率
      print('Validation error: %.1f%%' % error_rate(
          eval_in_batches(validation_data, sess), validation_labels))

  # 训练完成，打印模型在测试集上的结果
  test_error = error_rate(eval_in_batches(test_data, sess),
                          test_labels)
  print('Test error: %.1f%%' % test_error)

# 测试集数据看看
test_images = test_data[:EVAL_BATCH_SIZE, ...]

# 使用模型识别
test_predictions = sess.run(
eval_prediction,
feed_dict={eval_data: test_images})
predictions = np.argmax(test_predictions, 1)

# 显示4个结果
for i in range(4):
plt.imshow(np.reshape(test_images[i], [28, 28]), cmap='gray')
plt.show()
print("Model prediction:", predictions[i])