PyTorch

6 minute read

Published: August 01, 2022

This lesson covers PyTorch Tutorial, https://pytorch.org/tutorials/beginner/basics/intro.html

Tensors

topic = "pytorch"
lesson = 2

from n import *
home, models_path = get_project_dir("FashionMNIST")
print(home)

/home/naneja/datasets/n/FashionMNIST

import torch
import numpy as np

Initializing a Tensor

# directly from data
data = [[1, 2], [3, 4]]

x_data = torch.tensor(data)
print_(f"x_data={x_data}")

x_data = tensor([[1, 2], [3, 4]])

# from NumPy array

data = [[1, 2], [3, 4]]

np_array = np.array(data)
print_(f"np_array={np_array}")

x_np = torch.from_numpy(np_array)
print_(f"x_np={x_np}")

np_array = [[1 2] [3 4]]

x_np = tensor([[1, 2], [3, 4]])

# from another array

data = [[1, 2], [3, 4]]

x_data = torch.tensor(data)
print_(f"x_data={x_data}")

# retains the properties (shape, datatype)
x_ones = torch.ones_like(x_data)

# retains shape but overrides the datatype
x_rand = torch.rand_like(x_data, dtype=torch.float) 


print_("Ones Tensor:", x_ones)
#print(x_ones)

print_("Random Tensor:", x_rand)
#print(x_rand)

x_data = tensor([[1, 2], [3, 4]])

Ones Tensor:

tensor([[1, 1],
        [1, 1]])

Random Tensor:

tensor([[0.5770, 0.4122],
        [0.3629, 0.4553]])

# With random or constant values
shape = (2,3,)

rand_tensor = torch.rand(shape)
ones_tensor = torch.ones(shape)
zeros_tensor = torch.zeros(shape)

print_("Random Tensor:", rand_tensor)

print_("Ones Tensor:", ones_tensor)

print_("Zeros Tensor:", zeros_tensor)

Random Tensor:

tensor([[0.7589, 0.4365, 0.9772],
        [0.6484, 0.0067, 0.0211]])

Ones Tensor:

tensor([[1., 1., 1.],
        [1., 1., 1.]])

Zeros Tensor:

tensor([[0., 0., 0.],
        [0., 0., 0.]])

Attributes of a Tensor

shape
datatype
device on which they are stored

tensor = torch.rand(3,4)

print_(f"Shape of tensor: {tensor.shape}")
print_(f"Datatype of tensor: {tensor.dtype}")
print_(f"Device tensor is stored on: {tensor.device}")

Shape of tensor: torch.Size([3, 4])

Datatype of tensor: torch.float32

Device tensor is stored on: cpu

Operations on Tensors

https://pytorch.org/docs/stable/torch.html

print_("https://pytorch.org/docs/stable/torch.html")

https://pytorch.org/docs/stable/torch.html

# Move to the GPU if available
tensor = torch.rand(3,4)
print_(f"tensor", tensor)

if torch.cuda.is_available():
    tensor = tensor.to("cuda")
print_(f"tensor", tensor)

tensor = tensor.to("cpu")
print_(f"tensor", tensor)

tensor

tensor([[0.5592, 0.6119, 0.9760, 0.5708],
        [0.1763, 0.5915, 0.8959, 0.7148],
        [0.2837, 0.7816, 0.9187, 0.6705]])

tensor

tensor([[0.5592, 0.6119, 0.9760, 0.5708],
        [0.1763, 0.5915, 0.8959, 0.7148],
        [0.2837, 0.7816, 0.9187, 0.6705]], device='cuda:0')

tensor

tensor([[0.5592, 0.6119, 0.9760, 0.5708],
        [0.1763, 0.5915, 0.8959, 0.7148],
        [0.2837, 0.7816, 0.9187, 0.6705]])

# Standard numpy-like indexing and slicing:

tensor = torch.ones(4, 4)

print_("tensor", tensor)

print_(f"First row: {tensor[0]}")

print_(f"First column: {tensor[:, 0]}") # column = 0

print_(f"Last column: {tensor[..., -1]}") # column = -1

tensor[:, 1] = 0 # 1st index 1 and any zeroth index = 0

print_("tensor", tensor)

tensor

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

First row: tensor([1., 1., 1., 1.])

First column: tensor([1., 1., 1., 1.])

Last column: tensor([1., 1., 1., 1.])

tensor

tensor([[1., 0., 1., 1.],
        [1., 0., 1., 1.],
        [1., 0., 1., 1.],
        [1., 0., 1., 1.]])

# Joining tensors

tensor = torch.ones(4, 4)

t1 = torch.cat([tensor, tensor, tensor], dim=1)

t0 = torch.cat([tensor, tensor, tensor], dim=0)

print_("tensor", tensor)

print_("dim=1", t1)

print_("dim=0", t0)

tensor

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

dim = 1

tensor([[1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],
        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],
        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],
        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.]])

dim = 0

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

Arithmetic operations

# Matrix Multiplications

tensor = torch.ones(4, 4)
print_("tensor")
print(tensor)

# Matrix multiplication between two tensors
y1 = tensor @ tensor.T
print_("tensor @ tensor.T")
print(y1)

y2 = tensor.matmul(tensor.T)
print_("tensor.matmul(tensor.T)")
print(y2)


out = torch.rand(4,4)
y3 = torch.matmul(tensor, tensor.T, out=out)
print_("torch.matmul(tensor, tensor.T, out=torch.rand(4,4))")
print(y3)

tensor

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

tensor @ tensor.T

tensor([[4., 4., 4., 4.],
        [4., 4., 4., 4.],
        [4., 4., 4., 4.],
        [4., 4., 4., 4.]])

tensor.matmul(tensor.T)

tensor([[4., 4., 4., 4.],
        [4., 4., 4., 4.],
        [4., 4., 4., 4.],
        [4., 4., 4., 4.]])

torch.matmul(tensor, tensor.T, out = torch.rand(4,4))

tensor([[4., 4., 4., 4.],
        [4., 4., 4., 4.],
        [4., 4., 4., 4.],
        [4., 4., 4., 4.]])

# Element-wise product

tensor = torch.ones(4, 4)
print_("tensor")
print(tensor)

z1 = tensor * tensor
print_("tensor * tensor")
print(z1)

z2 = tensor.mul(tensor)
print_("tensor.mul(tensor)")
print(z2)

out = torch.rand(4,4)
z3 = torch.mul(tensor, tensor, out=out)
print_("torch.mul(tensor, tensor, out=torch.rand(4,4))")
print(z3)

tensor

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

tensor * tensor

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

tensor.mul(tensor)

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

torch.mul(tensor, tensor, out = torch.rand(4,4))

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

Single-element tensors

tensor = torch.ones(4, 4)
print_("tensor")
print(tensor)

# One element tensor
print_("tensor.sum()")
print(tensor.sum())

# One element tensor converted to numpy
print_("tensor.sum().item()")
print(tensor.sum().item())

tensor

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

tensor.sum()

tensor(16.)

tensor.sum().item()

16.0

In-place Operations

save some memory, but can be problematic when computing derivatives because of an immediate loss of history
Hence, their use is discouraged

tensor = torch.ones(4, 4)
print_("tensor")
print(tensor)

tensor = tensor.add(3)
print_("tensor")
print(tensor)

tensor.add_(3)
print_("tensor")
print(tensor)

tensor

tensor([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]])

tensor

tensor([[4., 4., 4., 4.],
        [4., 4., 4., 4.],
        [4., 4., 4., 4.],
        [4., 4., 4., 4.]])

tensor

tensor([[7., 7., 7., 7.],
        [7., 7., 7., 7.],
        [7., 7., 7., 7.],
        [7., 7., 7., 7.]])

Bridge with NumPy

Tensors on the CPU and NumPy arrays can share their underlying memory locations, and changing one will change the other

t = torch.ones(5)
print_(f"t = {t}")

n = t.numpy()
print_(f"n = {n}")

# change in the tensor reflects in the NumPy array
t.add_(1)

print_(f"t = {t}")
print_(f"n = {n}")

t = tensor([1., 1., 1., 1., 1.])

n = [1. 1. 1. 1. 1.]

t = tensor([2., 2., 2., 2., 2.])

n = [2. 2. 2. 2. 2.]

# NumPy array to Tensor

n = np.ones(5)
t = torch.from_numpy(n)

np.add(n, 3, out=n)

print_(f"n = {n}")
print_(f"t = {t}")

n = [4. 4. 4. 4. 4.]

t = tensor([4., 4., 4., 4., 4.], dtype = torch.float64)

n = np.ones(5)
t = torch.from_numpy(n)

t.add_(3)

print_(f"n = {n}")
print_(f"t = {t}")

n = [4. 4. 4. 4. 4.]

t = tensor([4., 4., 4., 4., 4.], dtype = torch.float64)

Share on

Twitter Facebook LinkedIn

PyTorch

Tensors

Initializing a Tensor

Attributes of a Tensor

Operations on Tensors

Arithmetic operations

Single-element tensors

In-place Operations

Bridge with NumPy

Share on

You May Also Enjoy

Applied Software Design

Code: CMake and Catch2

C++

Pointers: slide 1

C++

Arrays and Vectors: slide 1

C++

Functions: slide 1