Perceptron

Source code

Learn more about perceptrons

Import.

%matplotlib inline
import math 
import copy
import numpy as np
import pandas as pd
from matplotlib import pylab as plt

Create data.

df = pd. DataFrame(
    {
        'x1': [1.5, 2, 3, 1.5, 0.5, -1, -2, -3, -1.5, 0],
        'x2': [1, 2.5, 3, -2, 2, -3, -1.2, -0.5, 2, -1.5],
        'label': ['A', 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'B'],
        'label_index': [1.0, 1.0, 1.0, 1.0, 1.0, -1.0, -1.0, -1.0, -1.0, -1.0]
     }
)
df

Plot.

label_a = df[df['label_index'] == 1]
plt.scatter(label_a['x1'], label_a['x2'], label='Label A (1)', marker='o')

label_b = df[df['label_index'] == -1]
plt.scatter(label_b['x1'], label_b['x2'], label='Label B (-1)', marker='x')

line_x = np.linspace(-4, 4, 4)
plt.plot(line_x, line_x * -1, 'r-')

plt.xlabel('x1')
plt.ylabel('x2')
plt.xlim([-3.5, 3.5])
plt.ylim([-3.5, 3.5])
plt.legend()

Encode the discriminating function

$$
y(x) = f(w^T phi (x))
$$

def discriminant(p, w):
  """
  :param: p: feature vector
  :param: w: coefficient vector
  """
  return np.dot(p, w)

Code the activation function

$$
f(a) =
begin{cases}
+1 (a geq 0) \
-1 (a leq 0)
end{cases}
$$

def activate(x):
  """
  :param: x: discriminant result
  """
  if -1 < x:
    return 1
  else:
    return -1

Predict.

data_size = len(df.index)

x = np.array(df.loc[:, ['x1', 'x2']])
w = [1,1]
label_answer = np.array(df['label_index'])
output_test = np.zeros(data_size)
label_test = np.zeros(data_size)
result = np.zeros(data_size)

for i in range(data_size):
  output_test[i] = discriminant(x[i], w)
  label_test[i] = activate(output_test[i])
  result[i] = label_answer[i] == label_test[i]

df_test = copy.deepcopy(df)
df_test['discriminant value'] = output_test
df_test['prediction label'] = label_test
df_test['prediction flag'] = result
df_test

Code parameter updates

$$
w^{i+1} = w^{i} + eta phi (n) t(n)
$$

Learn.

eta = 0.1

max_iter = 100

for i in range(max_iter):
  output_test = np.zeros(data_size)
  label_test = np.zeros(data_size)
  result = np.zeros(data_size)

for j in range(data_size):
    output_test[j] = discriminant(x[j], w)
    label_test[j] = activate(output_test[j])
    result[j] = label_answer[j] == label_test[j]

error = 0

for j in range(data_size):
    if result[j] == 0.0:
      error -= (output_test[j] * label_answer[j])

miss_classification = np.sum(result == 0.0)

if miss_classification == 0:
    break

for j in np.random.permutation(np.arange(data_size)):
    if result[j] == 0.0:
      w += eta * x[j] * label_answer[j]
      break

plt.xlabel('x1')
plt.ylabel('x2')
plt.xlim([-3.5, 3.5])
plt.ylim([-3.5, 3.5])

plt.scatter(label_a['x1'], label_a['x2'], label='Label A (1)', marker='o')
plt.scatter(label_b['x1'], label_b['x2'], label='Label B (-1)', marker='x')

line_x1 = np.linspace(-4, 4, 4)
plt.plot(line_x1, -1 * line_x1 * w[0] / w[1], 'r-')

plt.legend()

Least squares

Source code

Learn more about least squares

Import.

%matplotlib inline
import math
import copy
import numpy as np
import pandas as pd
from matplotlib import pylab as plt

Create data. It is the law of springs.

df = pd. DataFrame(
    {
        'weight_mass': [5, 10, 15, 20, 25, 30, 35, 40, 45, 50],
        'spring_length': [5.3, 5.7, 6.4, 6.9, 7.7, 8.2, 8.4, 9.8, 9.9, 10.7]
    }
)

Encoding two-section sum errors

$$
MSE = sum_{i=1}{n} (y_i – f(x_i))^2
$$

def rss(data):
  """
  :param data
  """
  result = 0

for i in data.iterrows():
    result += (i[1]['spring_length'] - i[1]['prediction']) ** 2

return result

Encode the least squares (regression line)

$$
f(x) = ax + b \
a = frac{Cov_{x,y}}{sigma _x^2} \
b = bar{y} – a bar{x}
$$

def cov(x, y):
  """
  :param: x: sample x
  :param: y: sample y
  :return: covariance
  """
  x_mean = np.mean(x)
  y_mean = np.mean(y)
  n = len(x)
  c = 0.0

for i in range(n):
    x_i = x[i]
    y_i = y[i]
    c += (x_i - x_mean) * (y_i - y_mean)

return c / n

def std(x):
  """
  :param x: sample
  :return: standard deviation
  """
  mu = np.mean(x)
  _std = 0.0
  n = len(x)

for i in range(n):
    _std += (x[i] - mu) ** 2

_std = _std / n

return np.sqrt(_std)

def slope(x, y):
  a = cov(x, y) / (std(x) ** 2)
  return a

def intercept(x, y):
  mean_x = np.array(x).mean()
  mean_y = np.array(y).mean()
  a = slope(x, y)
  b = mean_y - mean_x * a
  return b

Predict.

a = slope(df['weight_mass'], df['spring_length'])
b = intercept(df['weight_mass'], df['spring_length'])

x_data = np.arange(0, 60)
y_data = a * x_data + b

plt.scatter(df['weight_mass'], df['spring_length'])
plt.plot(x_data, y_data, 'r-')
plt.show()

Logistic regression

Source code

Learn more about logistic regression

Import.

%matplotlib inline
import math
import copy
import numpy as np
import pandas as pd
from matplotlib import pylab as plt

Code the sigmoid function

$$
varsigma (x) = frac{1}{1+exp(-x)}
$$

def std_sigmoid(x):
  """
  :param: x: x
  """
  return 1 / (1 + np.exp(-x))

t = np.linspace(-6, 6, 100)
plt.plot(t, std_sigmoid(t))
plt.xlabel('Input')
plt.ylabel('Output')
plt.plot(0, 0.5, 'o')
plt.annotate('0.5', xy=(0, 0.5), xytext=(2, 0.5), arrowprops=dict(arrowstyle='->'), fontsize=12)

Logistic regression and step functions

def step(x):
  """
  :param:: x: x
  """
  return np.array(x >= 0)

plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)

t = np.linspace(-6, 6, 100)
plt.plot(t, std_sigmoid(t))
plt.plot(0, 0.5, 'o')

plt.text(-6, 0.3, 'B is High')
plt.text(1.5, 0.7, 'A is High')
plt.annotate('50%', xy=(0, 0.5), xytext=(1, 0.5), arrowprops=dict(arrowstyle='->'))

plt.title('LogisticRegression (Sigmoid Function)')
plt.xlabel('Input: Inner product'r'$mathbf{w}^{mathrm{T}}phi(x)$'' value')
plt.ylabel('Output: Probability of label A')

plt.subplot(1, 2, 2)

t = np.linspace(-6, 6, 100)
plt.plot(t, step(t))

plt.text(-4, 0.3, 'Label B')
plt.text(2, 0.7, 'Label A')

plt.title('Perceptron (Step Function)')
plt.xlabel('Input: Inner product'r'$mathbf{w}^{mathrm{T}}phi(x)$'' value')
plt.ylabel('Output: Probability of label A')

Encode the discriminating function

$$
y(x) = ς ( w^T phi (x))
$$

def discriminant(p, w):
  """
  :param p: feature vector
  :param w: coeficient vector
  """
  return np.dot(p, w)

df = pd. DataFrame(
    {
        'x0': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        'x1': [1.5, 2, 3, 1.5, 0.5, -1, -2, -3, -1.5, 0],
        'x2': [1, 2.5, 3, -2, 2, -3, -1.2, -0.5, 2, -1.5],
        'label': ['A', 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'B'],
        'label_index': [1.0, 1.0, 1.0, 1.0, 1.0, 0, 0, 0, 0, 0]
     }
)

x = np.array(df.loc[:, ['x0', 'x1', 'x2']])
w = [0, 1, 1]
N = len(df.index)
output_label = np.zeros(N)
y = np.zeros(N)

for i in range(N):
  output_label[i] = discriminant(x[i], w)
  y[i] = std_sigmoid(output_label[i])

df_ = copy.deepcopy(df)
df_['discriminant value'] = output_label
df_['prediction value Y'] = y
df_

Draw an identification boundary.

plt.figure(figsize=(6, 5))
plt.xlim([-3.5, 3.5])
plt.ylim([-3.5, 3.5])
plt.title('Logistic Regression Result')

for i in range(N):
  if df.loc[i, 'label_index'] == 1.0:
    m = 'o'
  else:
    m = '^'

plt.scatter(
      df.loc[i, 'x1'],
      df.loc[i, 'x2'],
      s=40,
      label='Label A',
      marker=m,
      edgecolor='k',
      cmap='bwr',
      vmin=0,
      vmax=1
  )

plt.annotate('Y=0.5 (Identification boundary)', xy=(0, 0), xytext=(0.5, 0.5), arrowprops=dict(arrowstyle='->'))

line_x1 = np.linspace(-5, 5, 100)
plt.plot(line_x1, -1 * (line_x1 * w[1] + w[0]) / w[2], 'r-')

Pre-regression data, isn't it?

Create regression data.

label_answer = np.array(df['label_index'])
label_test = np.zeros(N)
result = np.zeros(N)
threshold = 0.5

for i in range(N):
  label_test[i] = y[i] >= threshold

result[i] = label_answer[i] == label_test[i]

df_ = copy.deepcopy(df_)
df_['discriminant value'] = output_label
df_['prediction value Y'] = y
df_['prediction label'] = label_test
df_['predictions correct or incorrect'] = result
df_

Create a logarithmic function to find the mean cross-entropy error.

def log(x):
  """
  :param: x: x
  """
  return math.log(max(x, 1.0E-20))

error = 0

for i in range(N):
  error -= label_answer[i] * log(y[i]) + (1 - label_answer[i]) * log(1 - y[i])

error / N

0.24087238242672498

Parameter update looks like this.

eta = 0.05

error = 0

for i in range(N):
  w -= eta * (y[i] - label_answer[i]) * x[i]

Actually classify.

def discriminant(p, w):
  """
  :param p: feature vector
  :param w: coeficient vector
  """
  return np.dot(p, w)

def std_sigmoid(x):
  """
  :param: x: x
  """
  return 1 / (1 + np.exp(-x))

def predict(p, w):
  """
  :param p: feature vector
  :param w: coeficient vector
  """
  return std_sigmoid(discriminant(p, w))

def log(x):
  """
  :param: x: x
  """
  return math.log(max(x, 1.0E-20))

df = pd. DataFrame(
    {
        'x0': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        'x1': [1.5, 2, 3, 1.5, 0.5, -1, -2, -3, -1.5, 0],
        'x2': [1, 2.5, 3, -2, 2, -3, -1.2, -0.5, 2, -1.5],
        'label': ['A', 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'B'],
        'label_index': [1.0, 1.0, 1.0, 1.0, 1.0, 0, 0, 0, 0, 0]
     }
)

label_answer = np.array(df['label_index'])
w = [1, 0, 0]
x = np.array(df.loc[:, ['x0', 'x1', 'x2']])
max_iter = 40
N = len(df.index)
output_label = np.zeros(N)
y = np.zeros(N)
cost_sum = np.zeros(max_iter)

for iter_  in range(max_iter):
  for i in np.random.permutation(np.arange(N)):
    y[i] = predict(x[i], w)
    w -= eta * (y[i] - label_answer[i]) * x[i]

error = 0

for i in range(N):
    error -= label_answer[i] * log(y[i]) + (1 - label_answer[i]) * log(1 - y[i])

cost_sum[iter_] = error / N

plt.title('Sample data and Identification boundary')
plt.xlabel('x1')
plt.ylabel('x2')
plt.xlim([-5, 5])
plt.ylim([-5, 5])

label_a = df[df['label_index'] == 1]
plt.scatter(label_a['x1'], label_a['x2'], label='Label A', marker='o')
label_b = df[df['label_index'] == 0]
plt.scatter(label_b['x1'], label_b['x2'], label='Label B', marker='^')
plt.legend()

line_x1 = np.linspace(-5, 5, 10)
plt.plot(line_x1, -1 * (line_x1 * w[1] + w[0]) / w[2], 'r-')

plt.figure()
plt.title('Mean cross entropy error')
x_num = np.arange(max_iter)
plt.plot(x_num, cost_sum[x_num], 'r-')
plt.xlabel('epoch')
plt.ylabel('Mean error')