From 75824ff5ac3750959f6c2bbaecd4625d62ba9642 Mon Sep 17 00:00:00 2001 From: Freezed <2160318-free_zed@users.noreply.gitlab.com> Date: Tue, 20 Sep 2022 23:58:41 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20a=20basic=20OpenSearch=20clie?= =?UTF-8?q?nt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1st command: list the stream mapping --- .gitignore | 3 ++ Makefile | 25 +++++++++++++++ README.md | 43 +++++++++++++++++++++++++ client.sample.py | 24 ++++++++++++++ main.py | 76 ++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 7 ++++ requirements-dev.txt | 5 +++ requirements.txt | 2 ++ 8 files changed, 185 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 client.sample.py create mode 100755 main.py create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c3bbc60 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +client.py +__pycache__/ +issues.csv diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..90dd5c9 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +help: # Print help on Makefile + @grep '^[^.#]\+:\s\+.*#' Makefile | \ + sed "s/\(.\+\):\s*\(.*\) #\s*\(.*\)/`printf "\033[93m"`\1`printf "\033[0m"` \3/" | \ + expand -t20 + +open_all: # Open all projects files + ${EDITOR} ${VIRTUAL_ENV}/bin/activate + ${EDITOR} .gitignore client* main.py Makefile pyproject.toml README.md requirements-dev.txt requirements.txt + ${EDITOR} .git/hooks/p*-commit + +pre_commit: # Run the pre-commit hook + .git/hooks/pre-commit + +clean: # Remove files not tracked in source control + rm -rf .pytest_cache + find . -type f -name "*.orig" -delete + find . -type f -name "*.pyc" -delete + find . -type d -name "__pycache__" -delete + find . -type d -empty -delete + +lint: # Lint code + ${VIRTUAL_ENV}/bin/black --check --quiet *.py + ${VIRTUAL_ENV}/bin/pflake8 --config=pyproject.toml + ${VIRTUAL_ENV}/bin/pydocstyle + ${VIRTUAL_ENV}/bin/pylint --rcfile=pyproject.toml *.py diff --git a/README.md b/README.md index a447994..a310c04 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,46 @@ An [OVH Log Data Platform](https://docs.ovh.com/fr/logs-data-platform/) python c 1. Access LDP stream 1. Query the logs 1. Trigger actions + + +🚀 Use quick-start +----------------- + +Get the source & edit settings: +```bash +git clone git@gitlab.com:forga/tool/ovh/ldpy.git && cd ldpy +cp client.sample.py client.py +${EDITOR} client.py +``` + +Set up environment & run help: +```bash +mkdir ~/.venvs && python3 -m venv ~/.venvs/ldpy +source ~/.venvs/ldpy/bin/activate +pip install -r requirements.txt +python main.py -h +``` + +* Get stream mapping: `python main.py --mapping` + + +🚧 Devel quick-start +-------------------- + +- Built with `Python 3.8` +- Code linting with [`flake8`](https://pypi.org/project/flake8/), [`pylint`](https://pypi.org/project/pylint), [`black`](https://pypi.org/project/black) & [pydocstyle](https://pypi.org/project/pydocstyle/). +- Install development tools: + * `pip install -r requirements-dev.txt` +- A `Makefile` with tools : run `make help` to have a look +- Use the `pre-commit` git hook + * `echo "make --no-print-directory lint" > .git/hooks/pre-commit` + * `chmod u+x .git/hooks/pre-commit` + + +### 📌 Dependences + + +Details in [`requirements.txt`](requirements.txt) & [`requirements-dev.txt`](requirements-dev.txt) + +- [`opensearch-py`](https://opensearch.org/docs/latest/clients/python/) +- [`opensearch_dsl`](https://pypi.org/project/opensearch-dsl/) diff --git a/client.sample.py b/client.sample.py new file mode 100644 index 0000000..844fe9c --- /dev/null +++ b/client.sample.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# coding: utf8 + +"""Start OpenSearch client. + +Filename: client.py + +***REMOVED*** +""" + +from opensearchpy import OpenSearch + +LDP_ALIAS = "" +LDP_CLUSTER = "gra.logs.ovh.com" +LDP_CRED_TYPE = "token" +LDP_CRED_VALUE = "" +LDP_STREAM_NAME = "My awesome LDP stream" + +opnsrch_clt = OpenSearch( + LDP_CLUSTER, + http_auth=(LDP_CRED_VALUE, LDP_CRED_TYPE), + scheme="https", + port=9200, +) diff --git a/main.py b/main.py new file mode 100755 index 0000000..5211a64 --- /dev/null +++ b/main.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# coding: utf8 + +"""A basic opensearch.org client. + +All start here. + +Author: frederic zind 2022-08-02 +Licenses: GNU AGPL v3: http://www.gnu.org/licenses/ +""" + + +import argparse +import sys + +from pprint import pprint as pp + +from client import LDP_ALIAS, opnsrch_clt + + +class CustomFormatter( + argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter +): + """This convert file docstring into help text.""" + + pass + + +def parse_args(args=sys.argv[1:]): + """Argument parser. + + Put values & describe parameters here not in your code + Ease usage with help & docs + """ + parser = argparse.ArgumentParser( + description=sys.modules[__name__].__doc__, formatter_class=CustomFormatter + ) + + parser.add_argument( + "-m", + "--mapping", + action="store_true", + help="Show the stream mapping", + ) + + return parser.parse_args(args) + + +def get_map_props(): + """Get stream mapping. + + LDP indices are rolling and LDP do not implement the call + to get indices related to an alias. + """ + indices = list(opnsrch_clt.indices.get_alias("*")) + mapping = opnsrch_clt.indices.get_mapping(LDP_ALIAS)[indices[0]]["mappings"] + map_props = sorted(list(mapping["properties"])) + + return map_props + + +def main(options): + """Execute as a script. Functions related to the arguments passed.""" + if options.mapping: + response = get_map_props() + else: + raise RuntimeError + + return response + + +if __name__ == "__main__": + pargs = parse_args() + result = main(pargs) + + pp(result) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3fc10b8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.flake8] +max-line-length = 90 + +[tool.pylint.reports] +score = false +disable = ["W0107", "W0102"] +ignore = "client.sample.py" diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..7577837 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +black +flake8==4 # see: https://github.com/johnthagen/python-blueprint/issues/113 +pydocstyle +pylint +pyproject-flake8 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..551fc3c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +certifi +opensearch-py