Simple Slot Machine code chooses same option (Thompson Sampling)-CodePudding

Trying out this simple Slot Machine Code.

import numpy as np

slotConRates = [.02, .013, .013, .015, .018]
# Number of Tries
N = 10000
# Number of Machines
d = len(slotConRates)

# Add 1 if you get a Win in the Array X
X = np.zeros((N,d))
for i in range(N):
  for j in range(d):
    if np.random.rand() < slotConRates[j]:
      X[i][j] = 1

win_reward = np.zeros(d)
loss_reward = np.zeros(d)

# Taking our best Slot Machine through beta distribution and updating it's loss and wins

for i in range(N):
  selected = 0
  MaxRandom = 0
  for j in range(d):
    randomBeta = np.random.beta(win_reward[j]   1, loss_reward[j]   1)
    if randomBeta > MaxRandom:
      MaxRandom = randomBeta
      selected = j
      if X[i][selected] == 1:
        win_reward[selected]  = 1
      else:
        loss_reward[selected]  = 1
# Showing which slot machine is considered the best
nSelected = win_reward   loss_reward 
for i in range(d):
    print('Machine number '   str(i   1)   ' was selected '   str(nSelected[i])   ' times')
print('Conclusion: Best machine is machine number '   str(np.argmax(nSelected)   1))

It is always selecting the 1st machine the choice for every iteration though.

Machine number 1 was selected 10000.0 times
Machine number 2 was selected 1181.0 times
Machine number 3 was selected 1108.0 times
Machine number 4 was selected 640.0 times
Machine number 5 was selected 1314.0 times
Conclusion: Best machine is machine number 1

How should I fix this? The second for loop is the issue. Any insights into why this is happening?

CodePudding user response：

I don't really understand what you are doing, but I somehow think that this slight modification (change of indentation of last if-else-block) might be what you are looking for:

...
for i in range(N):
    selected = 0
    MaxRandom = 0
    for j in range(d):
        randomBeta = np.random.beta(win_reward[j]   1, loss_reward[j]   1)
        if randomBeta > MaxRandom:
            MaxRandom = randomBeta
            selected = j
    if X[i][selected] == 1:
        win_reward[selected]  = 1
    else:
        loss_reward[selected]  = 1
...

EDIT: Typical result after adjustment:

win_reward = array([ 34.,   7.,   3.,  20., 124.])
loss_reward = array([1733.,  656.,  431., 1271., 5721.])
nSelected = array([1767.,  663.,  434., 1291., 5845.])
Machine number 1 was selected 1767.0 times
Machine number 2 was selected 663.0 times
Machine number 3 was selected 434.0 times
Machine number 4 was selected 1291.0 times
Machine number 5 was selected 5845.0 times
Conclusion: Best machine is machine number 5
(Test sum over selections: 10000.0)

Complete listing that I have used:

import numpy as np

slotConRates = [.02, .013, .013, .015, .018]
N = 10000
d = len(slotConRates)

X = np.zeros((N,d))
for i in range(N):
    for j in range(d):
        if np.random.rand() < slotConRates[j]:
            X[i][j] = 1

win_reward = np.zeros(d)
loss_reward = np.zeros(d)

for i in range(N):
    selected = 0
    MaxRandom = 0
    for j in range(d):
        randomBeta = np.random.beta(win_reward[j]   1, loss_reward[j]   1)
        if randomBeta > MaxRandom:
            MaxRandom = randomBeta
            selected = j
    if X[i][selected] == 1:
        win_reward[selected]  = 1
    else:
        loss_reward[selected]  = 1

nSelected = win_reward   loss_reward 
print(f'{win_reward = }')
print(f'{loss_reward = }')
print(f'{nSelected = }')
for i in range(d):
    print(f'Machine number {i   1} was selected {nSelected[i]} times')
print(f'Conclusion: Best machine is machine number {np.argmax(nSelected)   1}')
print(f'(Test sum over selections: {nSelected.sum()})')