Unable to get >1000 events from API query

#!/usr/bin/python

import urllib
import urllib2
import json
import sys, getopt

ROUTERS = { 'MessagingRouter': 'messaging',
            'EventsRouter': 'evconsole',
            'ProcessRouter': 'process',
            'ServiceRouter': 'service',
            'DeviceRouter': 'device',
            'NetworkRouter': 'network',
            'TemplateRouter': 'template',
            'DetailNavRouter': 'detailnav',
            'ReportRouter': 'report',
            'MibRouter': 'mib',
            'ZenPackRouter': 'zenpack' }

class ZenossAPIUnsuccessful(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class ZenossAPI():
    def __init__(self, debug=False, **kwargs):
        self.ZENOSS_USERNAME = kwargs['ZENOSS_USERNAME']
        self.ZENOSS_PASSWORD = kwargs['ZENOSS_PASSWORD']
        self.ZENOSS_INSTANCE = kwargs['ZENOSS_INSTANCE']

        """
        Initialize the API connection, log in, and store authentication cookie
        """
        # Use the HTTPCookieProcessor as urllib2 does not save cookies by default
        self.urlOpener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
        if debug: self.urlOpener.add_handler(urllib2.HTTPHandler(debuglevel=1))
        self.reqCount = 1

        # Contruct POST params and submit login.
        loginParams = urllib.urlencode(dict(
                        __ac_name = self.ZENOSS_USERNAME,
                        __ac_password = self.ZENOSS_PASSWORD,
                        submitted = 'true',
                        came_from = self.ZENOSS_INSTANCE + '/zport/dmd'))
        self.urlOpener.open(self.ZENOSS_INSTANCE + '/zport/acl_users/cookieAuthHelper/login',
                            loginParams)

    def _router_request(self, router, method, data=[]):
        if router not in ROUTERS:
            raise Exception('Router "' + router + '" not available.')

        # Contruct a standard URL request for API calls
        req = urllib2.Request(self.ZENOSS_INSTANCE + '/zport/dmd/' +
                              ROUTERS[router] + '_router')

        # NOTE: Content-type MUST be set to 'application/json' for these requests
        req.add_header('Content-type', 'application/json; charset=utf-8')

        # Convert the request parameters into JSON
        reqData = json.dumps([dict(
                    action=router,
                    method=method,
                    data=data,
                    type='rpc',
                    tid=self.reqCount)])

        # Increment the request count ('tid'). More important if sending multiple
        # calls in a single request
        self.reqCount += 1

        #print self.urlOpener.open(req, reqData).read()
        # Submit the request and convert the returned JSON to objects
        return json.loads(self.urlOpener.open(req, reqData).read())


    def create_event_on_device(self, device, summary, severity, component, evclasskey, evclass):
        if severity not in ('Critical', 'Error', 'Warning', 'Info', 'Debug', 'Clear'):
            raise Exception('Severity "' + severity +'" is not valid.')

        data = dict(device=device, summary=summary, severity=severity,
                    component=component, evclasskey=evclasskey, evclass=evclass)
        return self._router_request('EventsRouter', 'add_event', [data])

    # def query_archive(self, page=None, limit=0, start=0, sort='lastTime', dir='desc', params=None, exclusion_filter=None, keys=None, uid=None, detailFormat=False):
    #     data = dict(page=page, limit=limit, start=start,
    #             sort=sort, dir=dir, params=params, exclusion_filter=exclusion_filter,
    #             keys=keys, uid=uid, detailFormat=detailFormat)
    #     return self._router_request('EventsRouter', 'queryArchive', [data])

    def query_events(self, limit=0, start=0, sort='lastTime', dir='desc', params=None, exclusion_filter=None, keys=None, page=None, archive=False, uid=None, detailFormat=False):
        data = dict(page=page, limit=limit, start=start,
                sort=sort, dir=dir, params=params, exclusion_filter=exclusion_filter,
                keys=keys, uid=uid, detailFormat=detailFormat, archive=archive)
        return self._router_request('EventsRouter', 'query', [data])
​

#!/usr/bin/python

import json
from ZenossAPIPack.ZenossAPI import ZenossAPI
from ZenossAPIPack.ZenossAPI import ZenossAPIUnsuccessful
import time

status = {
    "New": 0,
    "Ackd": 1,
    "Suppressed": 2,
    "Closed": 3,
    "Cleared": 4,
    "Dropped": 5,
    "Aged" : 6
}

severities = {
    0: "Clear",
    1: "Debug",
    2: "Info",
    3: "Warning",
    4: "Error",
    5: "Critical"
}

ZENOSS_INSTANCE = "http://<labIp>:8080" # Zenoss LAB
ZENOSS_USERNAME = "#####"
ZENOSS_PASSWORD = "#####"

#Statuses: 0: New, 1: Ackd, 2: Suppressed, 3: Closed, 4: Cleared, 5: Dropped, 6: Aged
#Severities: 0: Clear, 1: Debug, 2: Info, 3: Warning, 4: Error, 5: Critical

kwargs = {"ZENOSS_INSTANCE": ZENOSS_INSTANCE,
          "ZENOSS_USERNAME": ZENOSS_USERNAME,
          "ZENOSS_PASSWORD": ZENOSS_PASSWORD}

z = ZenossAPI(**kwargs)

def write_event_results(results, file): <snip>

current_query_params = {
    "limit": 9999,
    "params": {"eventState": [0,1,2,3,4,5], "firstTime": "2018-10-01 00:00:00"}
}

arch_query_params = {
    "limit": 9999,
    "params": {"eventState": [0,1,2,3,4,5], "firstTime": "2018-10-01 00:00:00"},
    "archive": True
}

query_results = z.query_events(**current_query_params)
print "Total Console Events: " + str(query_results['result']['totalCount'])
print "Total Console Events in Results: " + str(len(query_results['result']['events']))
event_results = query_results['result']['events']

write_event_results(event_results, file)

query_results = ''
event_results = ''
query_results = z.query_events(**arch_query_params)
print "Total Archived Events: " + str(query_results['result']['totalCount'])
print "Total Archived Events in Results: " + str(len(query_results['result']['events']))
write_event_results(event_results, file)

​

Output of command:

[aculbertson@zenoss-01.lab1 zenossAPI]$ ./queryEvents.py
Total Console Events: 26437
Total Console Events in Results: 1000
Total Archived Events: 64801
Total Archived Events in Results: 1000​​

Total Console Events: 427
Total Console Events in Results: 427
​Total Archived Events: 67722
Total Archived Events in Results: 67722

#!/bin/env python

import zenApiLib

evtRouter = zenApiLib.zenConnector(routerName = 'EventsRouter')
# Override default timeout(5s), event queries can take a bit of time to return
evtRouter.config['timeout'] = 45

current_query_params = {
    "limit": 1000,
    "params": {"eventState": [0,1,2,3,4,5], "firstTime": "2018-10-01 00:00:00"}
}

arch_query_params = {
    "limit": 1000,
    "params": {"eventState": [0,1,2,3,4,5], "firstTime": "2018-10-01 00:00:00"},
    "archive": True
}

event_results = []
for pagedResults in evtRouter.pagingMethodCall('query', **current_query_params):
    if not pagedResults['result']['success']:
        raise Exception(pagedResults['msg'])
    # Useful event processing
    for event in pagedResults['result']['events']:
        pass
    # Instead of 'Useful event processing', for this example just updating a total list of events...
    event_results += pagedResults['result']['events']
#
print "Total Console Events: {}".format(pagedResults['result']['totalCount'])
print "Total Console Events in Results: {}".format(len(event_results))

event_results = []
for pagedResults in evtRouter.pagingMethodCall('query', **arch_query_params):
    if not pagedResults['result']['success']:
        raise Exception(pagedResults['msg'])
    # Useful event processing
    for event in pagedResults['result']['events']:
        pass
    # Instead of 'Useful event processing', for this example just updating a total list of events...
    event_results += pagedResults['result']['events']
#
print "Total Archived Events: {}".format(pagedResults['result']['totalCount'])
print "Total Archived Events in Results: {}".format(len(event_results))
​