Skip to content

feat: paginate _list() across all OpenStack link styles #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Koan-Bot wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

feat: paginate _list() across all OpenStack link styles #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 69 additions & 3 deletions lib/OpenStack/MetaAPI/Roles/Listable.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,37 @@ sub _list {
my $extra_filters =
$self->api_specs()->query_filters_for('/get', $uri, $caller_args);

my $attribute = $extra[0];
my $opts = {};
if ($extra_filters) {
if (scalar @extra == 1) {
push @extra, {};
$opts = {};
} elsif (scalar @extra > 1) {
die "Too many args when calling _list for all...";
}
$extra[-1] = {%{$extra[-1]}, %$extra_filters};
$opts = {%$opts, %$extra_filters};
} elsif (scalar @extra > 1) {
$opts = $extra[-1] // {};
}

@all = $self->client->all($uri, @extra);
my $path = $uri;
while (defined $path) {
my $result = $self->client->get($path, %$opts);

unless (defined $result->{$attribute}) {
my $keys = join(', ', sort keys %$result);
die "Response from $path does not contain attribute "
. "'$attribute', possible options are $keys";
}

push @all, @{$result->{$attribute}};

$path = _extract_next_link($result, $attribute, $self->client->endpoint);

# only pass query opts on the first request; subsequent pages
# carry their own query string in the next link URL
$opts = {};
}
}

my @args = @$caller_args;
Expand Down Expand Up @@ -74,4 +95,49 @@ sub _list {
return @all;
}

# Extract the next page URL from an OpenStack paginated response.
# Supports three pagination styles:
# 1. <attribute>_links: [ {rel:"next", href:"..."} ] (Nova, Neutron)
# 2. links: { next: "..." } (Keystone)
# 3. next: "..." (Glance)
sub _extract_next_link {
my ($result, $attribute, $endpoint) = @_;

my $raw;

# Style 1: <attribute>_links array (e.g. servers_links, networks_links)
my $links_key = "${attribute}_links";
if (ref $result->{$links_key} eq 'ARRAY') {
for my $link (@{$result->{$links_key}}) {
if (ref $link eq 'HASH' && ($link->{rel} // '') eq 'next') {
$raw = $link->{href};
last;
}
}
return unless defined $raw;
}

# Style 2: links hash with next key (Keystone)
if (!defined $raw && ref $result->{links} eq 'HASH' && defined $result->{links}{next}) {
$raw = $result->{links}{next};
}

# Style 3: top-level next key (Glance)
$raw //= $result->{next};

return unless defined $raw;

# Normalize: strip the endpoint prefix from absolute URLs so the client
# doesn't double-prepend it (client->get() always prepends the endpoint)
if (defined $endpoint && $raw =~ m{^https?://}) {
# strip trailing slash from endpoint for matching
(my $base = $endpoint) =~ s{/+$}{};
if ($raw =~ s{^\Q$base\E}{}) {
# $raw is now a relative path like /servers?marker=...
}
}

return $raw;
}

1;
234 changes: 234 additions & 0 deletions t/list-pagination.t
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,234 @@
#!/usr/bin/env perl

use strict;
use warnings;

use OpenStack::MetaAPI ();

use Test2::Bundle::Extended;
use Test2::Tools::Explain;
use Test2::Plugin::NoWarnings;

use FindBin;
use lib "$FindBin::Bin/lib";

use Test::OpenStack::MetaAPI qw{:all};
use Test::OpenStack::MetaAPI::Auth qw{:all};

use JSON;

mock_lwp_useragent();

my $api = get_api_object(use_env => 0);
ok $api, "got one api object" or die;

# ---------------------------------------------------------------------------
# Test 1: Nova-style pagination (servers_links with rel:next)
# ---------------------------------------------------------------------------
{
note "Testing Nova-style pagination via servers_links";

# Page 1: two servers + a next link
mock_get_request(
'http://127.0.0.1:8774/v2.1/servers',
application_json(<<'JSON'),
{
"servers": [
{"id": "aaa", "name": "server-1"},
{"id": "bbb", "name": "server-2"}
],
"servers_links": [
{"rel": "next", "href": "http://127.0.0.1:8774/v2.1/servers?marker=bbb"}
]
}
JSON
);

# Page 2: one server, no next link
mock_get_request(
'http://127.0.0.1:8774/v2.1/servers?marker=bbb',
application_json(<<'JSON'),
{
"servers": [
{"id": "ccc", "name": "server-3"}
]
}
JSON
);

my @servers = $api->servers();
is scalar @servers, 3, "pagination collected all 3 servers across 2 pages";
is [map { $_->{id} } @servers], [qw(aaa bbb ccc)],
"servers returned in correct page order";
}

# ---------------------------------------------------------------------------
# Test 2: Single page (no pagination links) still works
# ---------------------------------------------------------------------------
{
note "Testing single-page response (no pagination)";

mock_get_request(
'http://127.0.0.1:8774/v2.1/servers',
application_json(<<'JSON'),
{
"servers": [
{"id": "xxx", "name": "only-server"}
]
}
JSON
);

my $server = $api->servers();
is $server->{id}, 'xxx', "single-page response returns one server";
}

# ---------------------------------------------------------------------------
# Test 3: servers_links present but no rel:next stops pagination
# ---------------------------------------------------------------------------
{
note "Testing servers_links with only self (no next)";

mock_get_request(
'http://127.0.0.1:8774/v2.1/servers',
application_json(<<'JSON'),
{
"servers": [
{"id": "111", "name": "s1"},
{"id": "222", "name": "s2"}
],
"servers_links": [
{"rel": "self", "href": "http://127.0.0.1:8774/v2.1/servers"}
]
}
JSON
);

my @servers = $api->servers();
is scalar @servers, 2, "stops when no next link in servers_links";
}

# ---------------------------------------------------------------------------
# Test 4: Pagination with client-side filtering
# ---------------------------------------------------------------------------
{
note "Testing pagination + client-side name filter";

mock_get_request(
'http://127.0.0.1:8774/v2.1/servers',
application_json(<<'JSON'),
{
"servers": [
{"id": "aaa", "name": "web-1"},
{"id": "bbb", "name": "db-1"}
],
"servers_links": [
{"rel": "next", "href": "http://127.0.0.1:8774/v2.1/servers?marker=bbb"}
]
}
JSON
);

mock_get_request(
'http://127.0.0.1:8774/v2.1/servers?marker=bbb',
application_json(<<'JSON'),
{
"servers": [
{"id": "ccc", "name": "web-2"}
]
}
JSON
);

my @servers = $api->servers(name => 'web-2');
is scalar @servers, 1, "filter applied after full pagination";
is $servers[0]->{id}, 'ccc', "correct server found on page 2";
}

# ---------------------------------------------------------------------------
# Test 5: Three pages of pagination
# ---------------------------------------------------------------------------
{
note "Testing three pages of pagination";

mock_get_request(
'http://127.0.0.1:8774/v2.1/servers',
application_json(<<'JSON'),
{
"servers": [
{"id": "p1", "name": "page1"}
],
"servers_links": [
{"rel": "next", "href": "http://127.0.0.1:8774/v2.1/servers?marker=p1"}
]
}
JSON
);

mock_get_request(
'http://127.0.0.1:8774/v2.1/servers?marker=p1',
application_json(<<'JSON'),
{
"servers": [
{"id": "p2", "name": "page2"}
],
"servers_links": [
{"rel": "next", "href": "http://127.0.0.1:8774/v2.1/servers?marker=p2"}
]
}
JSON
);

mock_get_request(
'http://127.0.0.1:8774/v2.1/servers?marker=p2',
application_json(<<'JSON'),
{
"servers": [
{"id": "p3", "name": "page3"}
]
}
JSON
);

my @servers = $api->servers();
is scalar @servers, 3, "three pages collected correctly";
is [map { $_->{id} } @servers], [qw(p1 p2 p3)],
"items from all three pages in order";
}

# ---------------------------------------------------------------------------
# Test 6: Network pagination (floatingips_links)
# ---------------------------------------------------------------------------
{
note "Testing Neutron-style pagination via floatingips_links";

mock_get_request(
'http://127.0.0.1:9696/v2.0/floatingips',
application_json(<<'JSON'),
{
"floatingips": [
{"id": "fip-1", "floating_ip_address": "10.0.0.1"}
],
"floatingips_links": [
{"rel": "next", "href": "http://127.0.0.1:9696/v2.0/floatingips?marker=fip-1"}
]
}
JSON
);

mock_get_request(
'http://127.0.0.1:9696/v2.0/floatingips?marker=fip-1',
application_json(<<'JSON'),
{
"floatingips": [
{"id": "fip-2", "floating_ip_address": "10.0.0.2"}
]
}
JSON
);

my @fips = $api->floatingips();
is scalar @fips, 2, "Neutron pagination collected both floating IPs";
}

done_testing;
Loading