Tranfer Learning Based on Pre-trained VGG16 Network

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import skimage.io
import skimage.transform
import urllib
import requests
from io import BytesIO
import flickrapi
import os

api_key = '******'
api_secret = '******'

flickr = flickrapi.FlickrAPI(api_key, api_secret)
keyword = 'car'
ph_car = flickr.walk(text=keyword, tag_mode='all', tags=keyword,extras='url_c',\
                     sort='relevance',per_page=100)
keyword = 'bicycle'
ph_bicycle = flickr.walk(text=keyword, tag_mode='all', tags=keyword,extras='url_c',\
                     sort='relevance',per_page=100)

# declare directory names
train_dir = 'train'
test_dir = 'test'
image_dir = ['car','bicycle']

# make a directories for the training and testing images
dir_path = []
for root_dir in [train_dir,test_dir]:
    # check if data exists
    if not os.path.isdir(root_dir):
        os.mkdir(root_dir)  
        
    for data_dir in image_dir:
        dir_path = root_dir + '/' + data_dir
        if not os.path.isdir(dir_path):
            os.mkdir(dir_path)
            print("Making directory %s" % dir_path)
        else:
            print("Will store %s images in directory %s" %(root_dir,dir_path))

import warnings
from random import seed
from random import random
    
ntrain = 1000 # number of training images to download
ntest = 300 # number of testing images to download

nrow = 64  # image dimension 1
ncol = 64  # image dimension 2

def download_img(photos, image_dir):
    keyword = image_dir
    # download images, first train then test
    seed(1) # make repeatable
    itrain = 0 # initialize counter
    itest = 0 # initialize counter
    inum = -1
    for photo in photos:
        url=photo.get('url_c')
        if not (url is None):

            # Create a file from the URL
            # This may only work in Python3
            response = requests.get(url)
            file = BytesIO(response.content)

            # Read image from file
            im = skimage.io.imread(file)

            # Resize images
            #im1 = skimage.transform.resize(im,(nrow,ncol),mode='constant')

            # Convert to uint8, suppress the warning about the precision loss
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                #im2 = skimage.img_as_ubyte(im1)
                im2 = skimage.img_as_ubyte(im)

            # Set the directory randomly 
            #if (random()< ntrain/(ntrain+ntest)):
            if (random()< (ntrain-itrain)/(ntrain-itrain + ntest-itest + 1e-15)):
                if (itrain<ntrain):
                    dir_path = train_dir + '/' + image_dir
                    i = itrain
                    itrain = itrain + 1
                elif (itest<ntest):
                    dir_path = test_dir + '/' + image_dir
                    i = itest
                    itest = itest + 1
                else:
                    break
            else:
                if (itest<ntest):
                    dir_path = test_dir + '/' + image_dir
                    i = itest
                    itest = itest + 1
                elif (itrain<ntrain):
                    dir_path = train_dir + '/' + image_dir
                    i = itrain
                    itrain = itrain + 1
                else:
                    break            
                
            # Save the image
            local_name = '{0:s}/{1:s}_{2:04d}.jpg'.format(dir_path,keyword,i)  
            skimage.io.imsave(local_name, im2)    
            inum += 1
            if inum%100 == 0:
                print(local_name)     

#call function
download_img(ph_car, 'car')   
download_img(ph_bicycle, 'bicycle')   

import torch
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader
import torchvision.transforms as transforms

# Create a transformation that converts to a tensor then and crops to a desired size (we'll do 64x64)
crop = transforms.RandomResizedCrop(64,scale=(0.5,1.0))
transform = transforms.Compose([crop, transforms.ToTensor(),
                               transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])

# Create a DataSet, using our transform and folder
train_ds = ImageFolder(root='train', transform=transform)
batch_size = 50
train_dl  = DataLoader(train_ds, batch_size=batch_size, shuffle=True)

# Create a DataSet, using our transform and folder
test_ds = ImageFolder(root='test', transform=transform)
batch_size = 50
test_dl  = DataLoader(test_ds, batch_size=batch_size)

import matplotlib.pyplot as plt
import numpy as np
%matplotlib inline

#### Code from https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html #####
def imshow(inp, title=None):
    """Imshow for Tensor."""
    inp = inp.numpy().transpose((1, 2, 0)) # rearrange dimensions from (color,y,x) -> (y,x,color)
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean # undo the normalization
    inp = np.clip(inp, 0, 1)
    # Display image, without ticks
    plt.imshow(inp)
    plt.xticks([])
    plt.yticks([])
    if title is not None:
        plt.title(title)

###########################################################################################

x, y = next(iter(train_dl))
from torchvision.utils import make_grid

%matplotlib inline
imshow(make_grid(x[0:8,:,:,:]),y[0:8])

import torch
from torchvision.models import vgg16

# Load the VGG16 network
model = vgg16(pretrained=True)
print(str(model))

# freeze layers
#print(model.features.parameters)
# ct = 0
# for child in model.children():
#     ct += 1
#     #print(ct)
#     if ct <= 2:
#         for param in child.parameters():
#             param.requires_grad = False

# or
#for param1 in model.features.parameters():
#     param1.requires_grad = False
# for param2 in model.avgpool.parameters():
#     param2.requires_grad = False

#or simply
for param in model.features.parameters():
    param.requires_grad = False
    

print(model.classifier[0])
#print(model.classifier)

import torch.nn as nn
num_classes = 1
# Replace the classifier part of the network
model.classifier = nn.Sequential(    
                nn.Linear(25088, 256),  
                nn.ReLU(),
                nn.Dropout(p=0.5),
                nn.Linear(256, num_classes),
                nn.Sigmoid(),
        )

print(str(model))

import torch.optim as optim

lrate = 1e-3
# initiate Adam optimizer
opt = optim.Adam(model.parameters(), lr=lrate)

criterion = nn.BCELoss()

epochs = 5
a_tr_loss = np.zeros([epochs])
a_tr_accuracy = np.zeros([epochs])
a_ts_loss = np.zeros([epochs])
a_ts_accuracy = np.zeros([epochs])

for epoch in range(epochs):
    correct = 0 # initialize error counter
    total = 0 # initialize total counter
    model.train() # put model in training mode
    
    batch_loss_tr = []
    # iterate over training set
    for train_iter, data in enumerate(train_dl):
        x_batch,y_batch = data
        out = model(x_batch)
        #y_batch = y_batch.to(dtype=torch.float)
        y_batch = y_batch.reshape(-1,1)
        # Compute Loss
        loss = criterion(out,y_batch.type(torch.float))
        batch_loss_tr.append(loss.item())
        # Zero gradients
        opt.zero_grad()
        # Compute gradients using back propagation
        loss.backward()
        # Take an optimization 'step'
        opt.step()
        
        # Compute Accuracy
        #print(out)   #the size of out = batch size!
        predict = out
        predict[predict>=0.5] = 1
        predict[predict<0.5] = 0
        predict = predict.to(dtype=torch.long)
        total += y_batch.size(0)
        correct += (predict == y_batch).sum().item()

    a_tr_loss[epoch] = np.mean(batch_loss_tr) # Compute average loss over epoch
    a_tr_accuracy[epoch] = 100*correct/total
    
    correct = 0
    total = 0
    batch_loss_ts = []
    model.eval() # put model in evaluation mode
    with torch.no_grad():
        for data in test_dl:
            images, labels = data
            labels = labels.reshape(-1,1)
            outputs = model(images)
            batch_loss_ts.append(criterion(outputs,labels.type(torch.float)).item())
            predicted = outputs
            predicted[predicted>=0.5] = 1
            predicted[predicted<0.5] = 0
            predicted = predicted.to(dtype=torch.long)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
    
    a_ts_loss[epoch] = np.mean(batch_loss_ts)
    a_ts_accuracy[epoch] = 100*correct/total
    
    # Print details every print_mod epoch
    print('Epoch: {0:2d}   Train Loss: {1:.3f}   '.format(epoch+1, a_tr_loss[epoch])
          +'Train Accuracy: {0:.2f}    Test Loss: {1:.3f}   '.format(a_tr_accuracy[epoch], a_ts_loss[epoch])
          +'Test Accuracy: {0:.2f}'.format(a_ts_accuracy[epoch]))
        
print('Done!')

# Save
PATH = 'saved_basic_model.pt'
torch.save(model.state_dict(), PATH)

n_show = 8

eg_dl = DataLoader(test_ds, batch_size=1, shuffle=True)

fig = plt.figure(figsize=(10,15))
with torch.no_grad():
    for i, data in enumerate(eg_dl):
        if i >= n_show:
            break        
        images, labels = data
        out = model(images)
        predicted = out
        predicted[predicted>=0.5] = 1
        predicted[predicted<0.5] = 0
        predicted = predicted.to(dtype=torch.long)    
        
        plt.subplot(4, 2, i+1)
        imshow(images[0])  
        plt.title('True label:'+str(int(labels[0]))+'; predicted label:'+str(int(predicted)))

    # iterate over training set
    for train_iter, data in enumerate(train_dl):
        x_batch,y_batch = data
        out = model(x_batch)
        #y_batch = y_batch.to(dtype=torch.float)
        y_batch = y_batch.reshape(-1,1)
        # Compute Loss
        loss = criterion(out,y_batch.type(torch.float))
        batch_loss_tr.append(loss.item())
        # Zero gradients
        opt.zero_grad()
        # Compute gradients using back propagation
        loss.backward()
        # Take an optimization 'step'
        opt.step()
        
        # Compute Accuracy
        #print(out)   #the size of out = batch size!
        predict = out
        predict[predict>=0.5] = 1
        predict[predict<0.5] = 0
        predict = predict.to(dtype=torch.long)
        total += y_batch.size(0)
        correct += (predict == y_batch).sum().item()

    # iterate over training set
    for train_iter, data in enumerate(train_dl):
        x_batch,y_batch = data
        y_batch = y_batch.view(-1,1)
        out = model(x_batch)
        # Compute Loss
        loss = criterion(out,y_batch.type(torch.float))
        batch_loss.append(loss.item())
        # Zero gradients
        opt.zero_grad()
        # Compute gradients using back propagation
        loss.backward()
        # Take an optimization 'step'
        opt.step()
        
        # 
        predicted = out.clamp(0,1).round().type(torch.long)
        total += y_batch.size(0)
        correct += (predicted == y_batch).sum().item()