From 13d44165de68e465a7b417e4a404a1eef07588bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C5=8Dan?= Date: Sat, 21 Mar 2026 09:53:41 -0600 Subject: [PATCH] feat: add pagination support to _list() for multi-page OpenStack responses Replace client->all() with a manual pagination loop that follows next-page links across all three OpenStack pagination styles: - _links array (Nova, Neutron) - links hash with next key (Keystone) - top-level next key (Glance) The previous implementation delegated to OpenStack::Client::all(), which only follows the top-level 'next' key. Most OpenStack services use _links arrays instead, causing result sets larger than the default page size to be silently truncated. Co-Authored-By: Claude Opus 4.6 --- lib/OpenStack/MetaAPI/Roles/Listable.pm | 72 +++++++- t/list-pagination.t | 234 ++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 3 deletions(-) create mode 100644 t/list-pagination.t diff --git a/lib/OpenStack/MetaAPI/Roles/Listable.pm b/lib/OpenStack/MetaAPI/Roles/Listable.pm index e26b1d7..0db8189 100644 --- a/lib/OpenStack/MetaAPI/Roles/Listable.pm +++ b/lib/OpenStack/MetaAPI/Roles/Listable.pm @@ -22,16 +22,37 @@ sub _list { my $extra_filters = $self->api_specs()->query_filters_for('/get', $uri, $caller_args); + my $attribute = $extra[0]; + my $opts = {}; if ($extra_filters) { if (scalar @extra == 1) { - push @extra, {}; + $opts = {}; } elsif (scalar @extra > 1) { die "Too many args when calling _list for all..."; } - $extra[-1] = {%{$extra[-1]}, %$extra_filters}; + $opts = {%$opts, %$extra_filters}; + } elsif (scalar @extra > 1) { + $opts = $extra[-1] // {}; } - @all = $self->client->all($uri, @extra); + my $path = $uri; + while (defined $path) { + my $result = $self->client->get($path, %$opts); + + unless (defined $result->{$attribute}) { + my $keys = join(', ', sort keys %$result); + die "Response from $path does not contain attribute " + . "'$attribute', possible options are $keys"; + } + + push @all, @{$result->{$attribute}}; + + $path = _extract_next_link($result, $attribute, $self->client->endpoint); + + # only pass query opts on the first request; subsequent pages + # carry their own query string in the next link URL + $opts = {}; + } } my @args = @$caller_args; @@ -74,4 +95,49 @@ sub _list { return @all; } +# Extract the next page URL from an OpenStack paginated response. +# Supports three pagination styles: +# 1. _links: [ {rel:"next", href:"..."} ] (Nova, Neutron) +# 2. links: { next: "..." } (Keystone) +# 3. next: "..." (Glance) +sub _extract_next_link { + my ($result, $attribute, $endpoint) = @_; + + my $raw; + + # Style 1: _links array (e.g. servers_links, networks_links) + my $links_key = "${attribute}_links"; + if (ref $result->{$links_key} eq 'ARRAY') { + for my $link (@{$result->{$links_key}}) { + if (ref $link eq 'HASH' && ($link->{rel} // '') eq 'next') { + $raw = $link->{href}; + last; + } + } + return unless defined $raw; + } + + # Style 2: links hash with next key (Keystone) + if (!defined $raw && ref $result->{links} eq 'HASH' && defined $result->{links}{next}) { + $raw = $result->{links}{next}; + } + + # Style 3: top-level next key (Glance) + $raw //= $result->{next}; + + return unless defined $raw; + + # Normalize: strip the endpoint prefix from absolute URLs so the client + # doesn't double-prepend it (client->get() always prepends the endpoint) + if (defined $endpoint && $raw =~ m{^https?://}) { + # strip trailing slash from endpoint for matching + (my $base = $endpoint) =~ s{/+$}{}; + if ($raw =~ s{^\Q$base\E}{}) { + # $raw is now a relative path like /servers?marker=... + } + } + + return $raw; +} + 1; diff --git a/t/list-pagination.t b/t/list-pagination.t new file mode 100644 index 0000000..3ef48d5 --- /dev/null +++ b/t/list-pagination.t @@ -0,0 +1,234 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use OpenStack::MetaAPI (); + +use Test2::Bundle::Extended; +use Test2::Tools::Explain; +use Test2::Plugin::NoWarnings; + +use FindBin; +use lib "$FindBin::Bin/lib"; + +use Test::OpenStack::MetaAPI qw{:all}; +use Test::OpenStack::MetaAPI::Auth qw{:all}; + +use JSON; + +mock_lwp_useragent(); + +my $api = get_api_object(use_env => 0); +ok $api, "got one api object" or die; + +# --------------------------------------------------------------------------- +# Test 1: Nova-style pagination (servers_links with rel:next) +# --------------------------------------------------------------------------- +{ + note "Testing Nova-style pagination via servers_links"; + + # Page 1: two servers + a next link + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "aaa", "name": "server-1"}, + {"id": "bbb", "name": "server-2"} + ], + "servers_links": [ + {"rel": "next", "href": "http://127.0.0.1:8774/v2.1/servers?marker=bbb"} + ] +} +JSON + ); + + # Page 2: one server, no next link + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers?marker=bbb', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "ccc", "name": "server-3"} + ] +} +JSON + ); + + my @servers = $api->servers(); + is scalar @servers, 3, "pagination collected all 3 servers across 2 pages"; + is [map { $_->{id} } @servers], [qw(aaa bbb ccc)], + "servers returned in correct page order"; +} + +# --------------------------------------------------------------------------- +# Test 2: Single page (no pagination links) still works +# --------------------------------------------------------------------------- +{ + note "Testing single-page response (no pagination)"; + + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "xxx", "name": "only-server"} + ] +} +JSON + ); + + my $server = $api->servers(); + is $server->{id}, 'xxx', "single-page response returns one server"; +} + +# --------------------------------------------------------------------------- +# Test 3: servers_links present but no rel:next stops pagination +# --------------------------------------------------------------------------- +{ + note "Testing servers_links with only self (no next)"; + + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "111", "name": "s1"}, + {"id": "222", "name": "s2"} + ], + "servers_links": [ + {"rel": "self", "href": "http://127.0.0.1:8774/v2.1/servers"} + ] +} +JSON + ); + + my @servers = $api->servers(); + is scalar @servers, 2, "stops when no next link in servers_links"; +} + +# --------------------------------------------------------------------------- +# Test 4: Pagination with client-side filtering +# --------------------------------------------------------------------------- +{ + note "Testing pagination + client-side name filter"; + + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "aaa", "name": "web-1"}, + {"id": "bbb", "name": "db-1"} + ], + "servers_links": [ + {"rel": "next", "href": "http://127.0.0.1:8774/v2.1/servers?marker=bbb"} + ] +} +JSON + ); + + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers?marker=bbb', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "ccc", "name": "web-2"} + ] +} +JSON + ); + + my @servers = $api->servers(name => 'web-2'); + is scalar @servers, 1, "filter applied after full pagination"; + is $servers[0]->{id}, 'ccc', "correct server found on page 2"; +} + +# --------------------------------------------------------------------------- +# Test 5: Three pages of pagination +# --------------------------------------------------------------------------- +{ + note "Testing three pages of pagination"; + + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "p1", "name": "page1"} + ], + "servers_links": [ + {"rel": "next", "href": "http://127.0.0.1:8774/v2.1/servers?marker=p1"} + ] +} +JSON + ); + + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers?marker=p1', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "p2", "name": "page2"} + ], + "servers_links": [ + {"rel": "next", "href": "http://127.0.0.1:8774/v2.1/servers?marker=p2"} + ] +} +JSON + ); + + mock_get_request( + 'http://127.0.0.1:8774/v2.1/servers?marker=p2', + application_json(<<'JSON'), +{ + "servers": [ + {"id": "p3", "name": "page3"} + ] +} +JSON + ); + + my @servers = $api->servers(); + is scalar @servers, 3, "three pages collected correctly"; + is [map { $_->{id} } @servers], [qw(p1 p2 p3)], + "items from all three pages in order"; +} + +# --------------------------------------------------------------------------- +# Test 6: Network pagination (floatingips_links) +# --------------------------------------------------------------------------- +{ + note "Testing Neutron-style pagination via floatingips_links"; + + mock_get_request( + 'http://127.0.0.1:9696/v2.0/floatingips', + application_json(<<'JSON'), +{ + "floatingips": [ + {"id": "fip-1", "floating_ip_address": "10.0.0.1"} + ], + "floatingips_links": [ + {"rel": "next", "href": "http://127.0.0.1:9696/v2.0/floatingips?marker=fip-1"} + ] +} +JSON + ); + + mock_get_request( + 'http://127.0.0.1:9696/v2.0/floatingips?marker=fip-1', + application_json(<<'JSON'), +{ + "floatingips": [ + {"id": "fip-2", "floating_ip_address": "10.0.0.2"} + ] +} +JSON + ); + + my @fips = $api->floatingips(); + is scalar @fips, 2, "Neutron pagination collected both floating IPs"; +} + +done_testing;