Thanks to Matiiss's answer, I managed to find a clue on how to do this:

import pygame

pygame.init()

size = (width, height) = (800, 600)
screen = pygame.display.set_mode(size)


class SceneElement:
    def __init__(self, x, y, width, height, color):
        self.x = x
        self.y = y
        self.width = width
        self.height = height
        self.color = color


class Scene:
    def __init__(self):
        self.elements = [
            SceneElement(150, 150, 200, 200, (55, 55, 10, 0.3)),
            SceneElement(250, 300, 200, 200, (155, 200, 10, 0.5)),
        ]

    def render(self, offset):
        screen.fill((255, 255, 255))
        for element in self.elements:
            x = element.x + offset[0]
            y = element.y + offset[1]
            pygame.draw.rect(screen, element.color, (x, y, element.width, element.height))


scene = Scene()
offset = pygame.Vector2()

running = True
while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        elif event.type == pygame.MOUSEWHEEL:
            print(event.x, event.y)
            if event.x == 0 and event.y == 1:
                direction = pygame.Vector2(event.x, event.y).normalize()
                offset += direction * 10
            elif event.x == 0 and event.y == -1:
                direction = pygame.Vector2(event.x, event.y).normalize()
                offset += direction * 10
            elif event.x == -1 and event.y == 0:
                direction = pygame.Vector2(1, event.y).normalize()
                offset += direction * 10
            elif event.x == 1 and event.y == 0:
                direction = pygame.Vector2(-1, event.y).normalize()
                offset += direction * 10

    scene.render(offset)
    pygame.display.flip()

pygame.quit()

This seems to work much more smoothly, but I feel like this could be improved still: https://i.sstatic.net/ZPNOO.jpg