When I use the Splunk API (from node.js) to query a given sid, I only get back 1000 results, even when supplying the count=0 argument. This particular sid happens to return a great many records -- over 6 million. When trying with an sid that returns much fewer records, say 5,000, they all are returned.
Here is my code:
var Request = require('request'); // 2.34.x
var options = {
url: 'htttps://splunksvr:8089/servicesNS/hector/search/search/jobs/[sid]/results?output_mode=json&count=0',
method: 'GET',
auth: {
user: 'hector',
pass: 'wouldntyouliketoknow'
},
rejectUnauthorized: false,
requestCert: true,
agent: false
};
Request(options, function(err, response, body) {
err && console.log('Error calling Splunk: ' + err);
body = JSON.parse(body);
body && body.results && console.log('query result count: ' + body.results.length);
});
Output:
query result count: 1000
I've also tried using pagination like this:
var Request = require('request');
var internals = {};
var options = {
url: 'htttps://splunksvr:8089/servicesNS/hector/search/search/jobs/[sid]/results?output_mode=json&count=300&offset={offset}',
method: 'GET',
auth: {
user: 'hector',
pass: 'wouldntyouliketoknow'
},
rejectUnauthorized: false,
requestCert: true,
agent: false
};
internals.querySplunk = function (options, offset, callback) {
options.originalUrl = options.url;
options.url = options.url.replace('{offset}', offset);
console.log(options.url);
Request(options, function(err, response, body) {
err && console.log('Error calling Splunk: ' + err);
body = JSON.parse(body);
var resultCount = body.results.length;
console.log('query result count: ' + resultCount);
if (resultCount === 0) {
console.log('Done getting results.');
return callback();
}
offset += 300;
options.url = options.originalUrl;
internals.querySplunk(options, offset, callback);
});
};
internals.querySplunk(options, 0, function () {
console.log('Exiting.');
});
Output: htttps://splunksvr:8089/servicesNS/hector/search/search/jobs/[sid]/results?output_mode=json&count=300&offset=0 query result count: 300 htttps://splunksvr:8089/servicesNS/hector/search/search/jobs/[sid]/results?output_mode=json&count=300&offset=300 query result count: 300 htttps://splunksvr:8089/servicesNS/hector/search/search/jobs/[sid]/results?output_mode=json&count=300&offset=600 query result count: 300 htttps://splunksvr:8089/servicesNS/hector/search/search/jobs/[sid]/results?output_mode=json&count=300&offset=900 query result count: 100 htttps://splunksvr:8089/servicesNS/hector/search/search/jobs/[sid]/results?output_mode=json&count=300&offset=1200 query result count: 0 Done getting results. Exiting.
As you can see, it still only pages to 1,000 results. What gives?