Source code for netlas.client

import requests
import orjson
import yaml
from io import TextIOWrapper

from netlas.exception import APIError
from netlas.helpers import check_status_code


[docs]class Netlas: def __init__(self, api_key: str = "", apibase: str = "https://app.netlas.io", debug: bool = False) -> None: """Netlas class constructor :param api_key: Personal API key, defaults to "" :type api_key: str :param apibase: Netlas API server address, defaults to "https://app.netlas.io" :type apibase: str, optional :param debug: Debug flag, defaults to False :type debug: bool, optional """ self.api_key: str = api_key self.apibase: str = apibase.rstrip("/") self.debug: bool = debug self.verify_ssl: bool = True if self.apibase != "https://app.netlas.io": self.verify_ssl = False if not self.api_key: raise APIError({"error": "API key is empty"}) self.headers = { 'Content-Type': 'application/json', 'X-Api-Key': self.api_key } def _request(self, endpoint: str = "/api/", params: object = {}) -> dict: """Private requests wrapper. Sends a request to Netlas API endpoint and process result. :param endpoint: API endpoint, defaults to "/api/" :type endpoint: str :param params: GET parameters for request, defaults to {} :type params: object, optional :raises APIError: Failed to parse JSON response :raises APIError: Other HTTP error :return: parsed JSON response :rtype: dict """ ret: dict = {} try: if not self.api_key: ret["error"] = "API key is empty" raise APIError(ret['error']) r = requests.get(f"{self.apibase}{endpoint}", params=params, headers=self.headers, verify=self.verify_ssl) response_data = orjson.loads(r.text) except orjson.JSONDecodeError: ret["error"] = "Failed to parse response data to JSON" if self.debug: ret["error_description"] = r.reason ret["error_data"] = r.text except requests.HTTPError: ret["error"] = f"{r.status_code}: {r.reason}" if self.debug: ret["error_description"] = r.reason ret["error_data"] = r.text if ret.get('error', None): raise APIError(ret['error']) check_status_code(request=r, debug=self.debug, ret=ret) ret = response_data return ret def _stream_request(self, endpoint: str = "/api/", params: object = {}) -> bytes: """Private stream requests wrapper. Sends a request to Netlas API endpoint and yield data from stream. :param endpoint: API endpoint, defaults to "/api/" :type endpoint: str :param params: GET parameters for request, defaults to {} :type params: object, optional :raises APIError: Failed to parse JSON response :raises APIError: Other HTTP error :return: Iterator of raw bytes from response :rtype: Iterator[bytes] """ ret: dict = {} if not self.api_key: ret["error"] = "API key is empty" raise APIError(ret['error']) try: with requests.get(f"{self.apibase}{endpoint}", params=params, headers=self.headers, verify=self.verify_ssl, stream=True) as r: check_status_code(request=r, debug=self.debug, ret=ret) for chunk in r.iter_content(chunk_size=2048): #skip keep-alive chunks if chunk: yield chunk except requests.HTTPError: ret["error"] = f"{r.status_code}: {r.reason}" if self.debug: ret["error_description"] = r.reason ret["error_data"] = r.text raise APIError(ret['error'])
[docs] def query(self, query: str, datatype: str = "response", indices: str = "") -> dict: """Send search query to Netlas API :param query: Search query string :type query: str :param datatype: Data type (choises: response, cert, domain), defaults to "response" :type datatype: str, optional :param indices: Comma-separated IDs of selected data indices (can be retrieved by `indices` method), defaults to "" :type indices: str, optional :return: search query result :rtype: dict """ endpoint = "/api/responses/" if datatype == "cert": endpoint = "/api/certs/" elif datatype == "domain": endpoint = "/api/domains/" ret = self._request( endpoint=endpoint, params={ "q": query, "indices": indices }, ) return ret
[docs] def count(self, query: str, datatype: str = "response", indices: str = "") -> dict: """Calculate total count of query string results :param query: Search query string :type query: str :param datatype: Data type (choises: response, cert, domain), defaults to "response" :type datatype: str, optional :param indices: Comma-separated IDs of selected data indices (can be retrieved by `indices` method), defaults to "" :type indices: str, optional :return: JSON object with total count of query string results :rtype: dict """ endpoint = "/api/responses_count/" if datatype == "cert": endpoint = "/api/certs_count/" elif datatype == "domain": endpoint = "/api/domains_count/" ret = self._request(endpoint=endpoint, params={ "q": query, "indices": indices }) return ret
[docs] def stat(self, query: str, indices: str = "") -> dict: """Get statistics of responses query string results :param query: Search query string :type query: str :param indices: Comma-separated IDs of selected data indices (can be retrieved by `indices` method), defaults to "" :type indices: str, optional :return: JSON object with statistics of responses query string results :rtype: dict """ ret = self._request( endpoint="/api/responses_stat/", params={ "q": query, "indices": indices }, ) return ret
[docs] def profile(self) -> dict: """Get user profile data :return: JSON object with user profile data :rtype: dict """ endpoint = "/api/users/profile/" ret = self._request(endpoint=endpoint) return ret
[docs] def host(self, host: str, hosttype: str = "ip", index: str = "") -> dict: """Get full information about host (ip or domain) :param host: IP or domain string :type host: str :param hosttype: `"ip"` or `"domain"`, defaults to "ip" :type hosttype: str, optional :param index: ID of selected data indices (can be retrieved by `indices` method), defaults to "" :type index: str, optional :return: JSON object with full information about host :rtype: dict """ endpoint = "/api/ip/" if hosttype == "domain": endpoint = "/api/domain/" ret = self._request( endpoint=endpoint, params={ "q": host, "index": index }, ) return ret
[docs] def download(self, query: str, datatype: str = "response", size: int = 10, indices: str = "") -> bytes: """Download data from Netlas :param query: Search query string :type query: str :param datatype: Data type (choises: response, cert, domain), defaults to "response" :type datatype: str, optional :param size: Download documents count, defaults to 10 :type size: int, optional :param indices: Comma-separated IDs of selected data indices (can be retrieved by `indices` method), defaults to "" :type indices: str, optional :return: Iterator of raw data :rtype: Iterator[bytes] """ endpoint = "/api/responses/download/" if datatype == "cert": endpoint = "/api/certs/download/" elif datatype == "domain": endpoint = "/api/domains/download/" for ret in self._stream_request( endpoint=endpoint, params={ "q": query, "size": size, "indices": indices }, ): yield ret
[docs] def indices(self) -> list: """Get available data indices :return: List of available indices :rtype: list """ endpoint = "/api/indices/" ret = self._request(endpoint=endpoint) return ret