Release v1.30 (#649)

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.2.1
+current_version = 1.3.0
 tag = False
 commit = False
 

.github/CODEOWNERS

@@ -1,5 +1,5 @@
 # Github codeowners file to improve our lives when selecting users for PR's
 
-* @bradchiappetta @elliottminns @Obsecurus @superducktoes @nathanqthai
+* @bradchiappetta @elliottminns @Obsecurus @superducktoes @nathanqthai @ianling
 
 /requirements/ @bradchiappetta

CHANGELOG.rst

@@ -2,6 +2,31 @@
 Changelog
 =========
 
+Version `1.3.0`_
+================
+**Date**: September 07, 2022
+
+* API client:
+
+  * Rework logging to remove structlog predefined logging to follow best practices
+  * Update the `quick` command to use a POST API request instead of a GET API request
+  * Fix bug with `ip_multi` preventing lists of more than 1000 entries from returning correct data
+  * Fix bug with `ip_multi` when cache is disabled not returning correct values
+  * Fix bug with `ip_multi` where `include_invalid` parameter was not working
+
+* CLI:
+
+  * None
+
+* Dependencies:
+
+  * Updated Click to 8.1.3
+  * Updated colorama to 0.4.5
+  * Updated jinja2 to 3.1.2
+  * Updated more-itertools to 8.14.0
+  * Update requests to 2.28.1
+  * Removed structlog
+
 Version `1.2.1`_
 ================
 **Date**: June 03, 2022
@@ -287,3 +312,4 @@ Version `0.2.0`_
 .. _`1.1.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v1.0.0...1.1.0
 .. _`1.2.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v1.1.0...1.2.0
 .. _`1.2.1`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v1.2.0...1.2.1
+.. _`1.3.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v1.2.1...1.3.0

docs/source/conf.py

@@ -25,7 +25,7 @@
 author = "GreyNoise Intelligence"
 
 # The full version, including alpha/beta/rc tags
-release = "1.2.1"
+release = "1.3.0"
 
 
 # -- General configuration ---------------------------------------------------

requirements/common.txt

@@ -1,15 +1,15 @@
-Click==8.1.2
+Click==8.1.3
 ansimarkup==1.4.0
 cachetools==5.2.0;python_version>='3'
-colorama==0.4.4
+colorama==0.4.5
 click-default-group==1.2.2
 click-repl==0.2.0
 dict2xml==1.7.1;python_version>='3'
 ipaddress==1.0.23
 jinja2==2.11.3;python_version=='3.5' # pyup: ignore
-jinja2==3.1.0;python_version>='3.6'
-more-itertools==8.13.0;python_version>='3'
-requests==2.27.1
+jinja2==3.1.2;python_version>='3.6'
+more-itertools==8.14.0;python_version>='3'
+requests==2.28.1
 six==1.16.0
 structlog==20.1.0;python_version=='3.5' # pyup: ignore
 structlog==21.5.0;python_version>='3.6'

requirements/dev.txt

@@ -3,4 +3,4 @@
 advbumpversion==1.2.0
 ipython==8.4.0;python_version>='3'
 pre-commit==2.20.0
-tox==3.25.1
+tox==3.26.0

requirements/test.txt

@@ -1,12 +1,13 @@
-black==22.6.0;python_version>='3.6'
+black==22.8.0;python_version>='3.6'
 flake8==3.9.2;python_version=='3.5' # pyup: ignore
-flake8==4.0.1;python_version>='3.6'
+flake8==5.0.4;python_version>='3.6'
 isort==4.3.21;python_version=='3.5' # pyup: ignore
 isort==5.10.1;python_version>='3.6'
 mock==3.0.5;python_version=='3.5' # pyup: ignore
 mock==4.0.3;python_version>='3.6'
 pylint==2.6.2;python_version=='3.5' # pyup: ignore
-pylint==2.12.2;python_version>='3.6' # pyup: ignore  ## remove once pyup is fixed or pylint releases 2.13.x or later
+pylint==2.13.9;python_version=='3.6'
+pylint==2.15.2;python_version>='3.7'
 pytest-cov==2.12.1;python_version=='3.5' # pyup: ignore
 pytest-cov==3.0.0;python_version>='3.6'
 pytest==6.1.2;python_version=='3.5' # pyup: ignore

setup.py

@@ -27,7 +27,7 @@ def read(fname):
 
 setup(
     name="greynoise",
-    version="1.2.1",
+    version="1.3.0",
     description="Abstraction to interact with GreyNoise API.",
     url="https://greynoise.io/",
     author="GreyNoise Intelligence",

src/greynoise/__version__.py

@@ -5,4 +5,4 @@
 __maintainer__ = "GreyNoise Intelligence"
 __email__ = "[email protected]"
 __status__ = "BETA"
-__version__ = "1.2.1"
+__version__ = "1.3.0"

src/greynoise/api/__init__.py

@@ -1,22 +1,20 @@
 """GreyNoise API client."""
 
+import logging
 import re
 from collections import OrderedDict
 
 import cachetools
 import more_itertools
 import requests
-import structlog
 
 from greynoise.__version__ import __version__
 from greynoise.api.analyzer import Analyzer
 from greynoise.api.filter import Filter
 from greynoise.exceptions import RateLimitError, RequestFailure
-from greynoise.util import configure_logging, load_config, validate_ip
+from greynoise.util import load_config, validate_ip
 
-if not structlog.is_configured():
-    configure_logging()
-LOGGER = structlog.get_logger()
+LOGGER = logging.getLogger(__name__)
 
 
 def initialize_cache(cache_max_size, cache_ttl):
@@ -170,15 +168,13 @@ def _request(self, endpoint, params=None, json=None, method="get"):
         else:
             url = "/".join([self.api_server, self.API_VERSION, endpoint])
 
-        LOGGER.debug(
-            "Sending API request...",
-            url=url,
-            method=method,
-            headers=headers,
-            params=params,
-            json=json,
-            proxy=self.proxy,
-        )
+        LOGGER.debug("Sending API request...URL: %s", url)
+        LOGGER.debug("Sending API request...method: %s", method)
+        LOGGER.debug("Sending API request...headers: %s", headers)
+        LOGGER.debug("Sending API request...params: %s", params)
+        LOGGER.debug("Sending API request...json: %s", json)
+        LOGGER.debug("Sending API request...proxy: %s", self.proxy)
+
         request_method = getattr(self.session, method)
         if self.proxy:
             proxies = {protocol: self.proxy for protocol in ("http", "https")}
@@ -200,12 +196,7 @@ def _request(self, endpoint, params=None, json=None, method="get"):
         else:
             body = response.text
 
-        LOGGER.debug(
-            "API response received",
-            status_code=response.status_code,
-            content_type=content_type,
-            body=body,
-        )
+        LOGGER.debug("API response received %s %s", response.status_code, body)
 
         if response.status_code == 429:
             raise RateLimitError()
@@ -268,9 +259,7 @@ def interesting(self, ip_address):
                 "message": "Interesting report not supported with Community offering"
             }
         else:
-            LOGGER.debug(
-                "Reporting interesting IP: %s...", ip_address, ip_address=ip_address
-            )
+            LOGGER.debug("Reporting interesting IP: %s...", ip_address)
             validate_ip(ip_address)
 
             endpoint = self.EP_INTERESTING.format(ip_address=ip_address)
@@ -287,7 +276,7 @@ def ip(self, ip_address):  # pylint: disable=C0103
         :rtype: dict
 
         """
-        LOGGER.debug("Getting context for %s...", ip_address, ip_address=ip_address)
+        LOGGER.debug("Getting context for %s...", ip_address)
         validate_ip(ip_address)
 
         if self.offering.lower() == "community":
@@ -325,13 +314,7 @@ def query(self, query, size=None, scroll=None):
         if self.offering == "community":
             response = {"message": "GNQL not supported with Community offering"}
         else:
-            LOGGER.debug(
-                "Running GNQL query: %s...",
-                query,
-                query=query,
-                size=size,
-                scroll=scroll,
-            )
+            LOGGER.debug("Running GNQL query: %s %s %s...", query, size, scroll)
             params = {"query": query}
             if size is not None:
                 params["size"] = size
@@ -361,7 +344,7 @@ def quick(self, ip_addresses, include_invalid=False):  # pylint: disable=R0912,R
             if isinstance(ip_addresses, str):
                 ip_addresses = ip_addresses.split(",")
 
-            LOGGER.debug("Getting noise status...", ip_addresses=ip_addresses)
+            LOGGER.debug("Getting noise status for %s...", ip_addresses)
 
             valid_ip_addresses = [
                 ip_address
@@ -388,7 +371,7 @@ def quick(self, ip_addresses, include_invalid=False):  # pylint: disable=R0912,R
                     )
                     for chunk in chunks:
                         api_result = self._request(
-                            self.EP_NOISE_MULTI, json={"ips": chunk}
+                            self.EP_NOISE_MULTI, method="post", json={"ips": chunk}
                         )
                         if isinstance(api_result, list):
                             api_results.extend(api_result)
@@ -408,7 +391,9 @@ def quick(self, ip_addresses, include_invalid=False):  # pylint: disable=R0912,R
                     valid_ip_addresses, self.IP_QUICK_CHECK_CHUNK_SIZE
                 )
                 for chunk in chunks:
-                    result = self._request(self.EP_NOISE_MULTI, json={"ips": chunk})
+                    result = self._request(
+                        self.EP_NOISE_MULTI, method="post", json={"ips": chunk}
+                    )
                     if isinstance(result, list):
                         results.extend(result)
                     else:
@@ -455,7 +440,7 @@ def ip_multi(self, ip_addresses, include_invalid=False):  # pylint: disable=R091
             if isinstance(ip_addresses, str):
                 ip_addresses = ip_addresses.split(",")
 
-            LOGGER.debug("Getting noise context...", ip_addresses=ip_addresses)
+            LOGGER.debug("Getting noise context for IPs: %s", ip_addresses)
 
             valid_ip_addresses = [
                 ip_address
@@ -494,10 +479,6 @@ def ip_multi(self, ip_addresses, include_invalid=False):  # pylint: disable=R091
                         else:
                             api_results.append(api_result)
 
-                        for ip_address in valid_ip_addresses:
-                            if ip_address not in api_results:
-                                api_results.append({"ip": ip_address, "seen": False})
-
                     for result in api_results:
                         ip_address = result["ip"]
 
@@ -514,8 +495,9 @@ def ip_multi(self, ip_addresses, include_invalid=False):  # pylint: disable=R091
                 )
                 for chunk in chunks:
                     result = self._request(
-                        self.EP_NOISE_CONTEXT_MULTI, json={"ips": chunk}
+                        self.EP_NOISE_CONTEXT_MULTI, method="post", json={"ips": chunk}
                     )
+                    result = result["data"]
                     if isinstance(result, list):
                         results.extend(result)
                     else:
@@ -524,7 +506,12 @@ def ip_multi(self, ip_addresses, include_invalid=False):  # pylint: disable=R091
             if include_invalid:
                 for ip_address in ip_addresses:
                     if ip_address not in valid_ip_addresses:
-                        results.append({"ip": ip_address, "seen": False})
+                        results.append(
+                            {
+                                "ip": ip_address,
+                                "noise": False,
+                            }
+                        )
 
         return results
 
@@ -533,7 +520,7 @@ def stats(self, query, count=None):
         if self.offering == "community":
             response = {"message": "Stats Query not supported with Community offering"}
         else:
-            LOGGER.debug("Running GNQL stats query: %s...", query, query=query)
+            LOGGER.debug("Running GNQL stats query: %s...", query)
             params = {"query": query}
             if count is not None:
                 params["count"] = count
@@ -571,7 +558,7 @@ def riot(self, ip_address):
         if self.offering == "community":
             response = {"message": "RIOT lookup not supported with Community offering"}
         else:
-            LOGGER.debug("Checking RIOT for %s...", ip_address, ip_address=ip_address)
+            LOGGER.debug("Checking RIOT for %s...", ip_address)
             validate_ip(ip_address)
 
             endpoint = self.EP_RIOT.format(ip_address=ip_address)

src/greynoise/cli/__init__.py

@@ -1,12 +1,10 @@
 """GreyNoise command line Interface."""
 
 import click
-import structlog
 from click_default_group import DefaultGroup
 from click_repl import register_repl
 
 from greynoise.cli import subcommand
-from greynoise.util import configure_logging
 
 
 @click.group(
@@ -16,8 +14,6 @@
 )
 def main():
     """GreyNoise CLI."""
-    if not structlog.is_configured():
-        configure_logging()
 
 
 SUBCOMMAND_FUNCTIONS = [

src/greynoise/cli/decorator.py

@@ -4,9 +4,9 @@
 
 """
 import functools
+import logging
 
 import click
-import structlog
 from requests.exceptions import RequestException
 
 from greynoise.api import GreyNoise
@@ -15,7 +15,7 @@
 from greynoise.exceptions import RequestFailure
 from greynoise.util import load_config
 
-LOGGER = structlog.get_logger()
+LOGGER = logging.getLogger(__name__)
 
 
 def echo_result(function):
@@ -70,12 +70,14 @@ def wrapper(*args, **kwargs):
             else:
                 error_message = "API error: {}".format(body)
             LOGGER.error(error_message)
-            click.echo(error_message)
             click.get_current_context().exit(-1)
         except RequestException as exception:
             error_message = "API error: {}".format(exception)
             LOGGER.error(error_message)
-            click.echo(error_message)
+            click.get_current_context().exit(-1)
+        except ValueError as exception:
+            error_message = "Validator error: {}".format(exception)
+            LOGGER.error(error_message)
             click.get_current_context().exit(-1)
 
     return wrapper