From 1aa2a1628939d8e9d9c43406a0d1cda8729d9c09 Mon Sep 17 00:00:00 2001 From: Mathieu Lecarme Date: Sun, 3 Apr 2022 12:24:03 +0200 Subject: [PATCH 1/5] Draft. --- contrib/meilisearch/.gitignore | 2 + contrib/meilisearch/Makefile | 12 ++++++ contrib/meilisearch/docker-compose.yml | 11 ++++++ contrib/meilisearch/index.py | 54 ++++++++++++++++++++++++++ contrib/meilisearch/requirements.txt | 3 ++ 5 files changed, 82 insertions(+) create mode 100644 contrib/meilisearch/.gitignore create mode 100644 contrib/meilisearch/Makefile create mode 100644 contrib/meilisearch/docker-compose.yml create mode 100755 contrib/meilisearch/index.py create mode 100644 contrib/meilisearch/requirements.txt diff --git a/contrib/meilisearch/.gitignore b/contrib/meilisearch/.gitignore new file mode 100644 index 0000000..9ee05fe --- /dev/null +++ b/contrib/meilisearch/.gitignore @@ -0,0 +1,2 @@ +venv +data.ms diff --git a/contrib/meilisearch/Makefile b/contrib/meilisearch/Makefile new file mode 100644 index 0000000..c084e76 --- /dev/null +++ b/contrib/meilisearch/Makefile @@ -0,0 +1,12 @@ + +venv: + python3 -m venv venv + ./venv/bin/pip install -U pip wheel + ./venv/bin/pip install -r requirements.txt + +up: + docker-compose up -d + docker ps + +pull: + docker-compose pull diff --git a/contrib/meilisearch/docker-compose.yml b/contrib/meilisearch/docker-compose.yml new file mode 100644 index 0000000..6b88b9b --- /dev/null +++ b/contrib/meilisearch/docker-compose.yml @@ -0,0 +1,11 @@ +--- + +version: "3" + +services: + meilisearch: + image: getmeili/meilisearch:latest + volumes: + - ./data.ms:/data.ms + ports: + - 7700:7700 diff --git a/contrib/meilisearch/index.py b/contrib/meilisearch/index.py new file mode 100755 index 0000000..4a98b72 --- /dev/null +++ b/contrib/meilisearch/index.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +from pathlib import Path +from io import StringIO + +import yaml +from markdown import markdown +from meilisearch import Client + + +def pages(): + "return all pages" + for readme in Path('.').glob('**/README.md'): + if readme == Path("README.md"): # it's the home + continue + with open(readme, 'r') as file: + head = StringIO() + body = StringIO() + state = None + for line in file: + if state is None and line == "---\n": + state = "head" + continue + if state == "head": + if line == "---\n": + state = "body" + continue + head.write(line) + else: + body.write(line) + if head.tell() == 0: # empty + continue + head.seek(0) + head = yaml.safe_load(head) + body.seek(0) + txt = markdown(body.read()) + yield str(readme), head['title'], txt + + +if __name__ == "__main__": + client = Client('http://127.0.0.1:7700') + try: + idx = client.get_index('pages') + except Exception: + client.create_index('pages', dict(primaryKey='path')) + print(idx) + + for path, title, body in pages(): + client.index('pages').add_documents([{ + 'path': path, + 'title': title, + 'body': body, + }]) + print(title) diff --git a/contrib/meilisearch/requirements.txt b/contrib/meilisearch/requirements.txt new file mode 100644 index 0000000..5fac84f --- /dev/null +++ b/contrib/meilisearch/requirements.txt @@ -0,0 +1,3 @@ +pyyaml +meilisearch +markdown From 952d9b3a45934285fef07338331b9cd82601086f Mon Sep 17 00:00:00 2001 From: Mathieu Lecarme Date: Sun, 3 Apr 2022 12:59:20 +0200 Subject: [PATCH 2/5] Cleaning id --- contrib/meilisearch/index.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/meilisearch/index.py b/contrib/meilisearch/index.py index 4a98b72..83c6cba 100755 --- a/contrib/meilisearch/index.py +++ b/contrib/meilisearch/index.py @@ -10,7 +10,7 @@ from meilisearch import Client def pages(): "return all pages" - for readme in Path('.').glob('**/README.md'): + for readme in Path('../..').glob('**/README.md'): if readme == Path("README.md"): # it's the home continue with open(readme, 'r') as file: @@ -34,7 +34,8 @@ def pages(): head = yaml.safe_load(head) body.seek(0) txt = markdown(body.read()) - yield str(readme), head['title'], txt + # removing ../../, README.md and replace / with _ + yield str(readme)[11:-10].replace("/", "_"), head['title'], txt if __name__ == "__main__": @@ -43,12 +44,14 @@ if __name__ == "__main__": idx = client.get_index('pages') except Exception: client.create_index('pages', dict(primaryKey='path')) - print(idx) - + print("the index is created") + else: + print("the index is already here") + for path, title, body in pages(): client.index('pages').add_documents([{ 'path': path, 'title': title, 'body': body, }]) - print(title) + print(title, path) From 196fceecb4b9c330780de36e0653a198dba29bb8 Mon Sep 17 00:00:00 2001 From: Mathieu Lecarme Date: Sun, 3 Apr 2022 12:59:45 +0200 Subject: [PATCH 3/5] =?UTF-8?q?Purge=20=F0=9F=94=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/meilisearch/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contrib/meilisearch/Makefile b/contrib/meilisearch/Makefile index c084e76..cc3fcc3 100644 --- a/contrib/meilisearch/Makefile +++ b/contrib/meilisearch/Makefile @@ -8,5 +8,13 @@ up: docker-compose up -d docker ps +down: + docker-compose down + +purge: + make down + rm -rf data.ms + make up + pull: docker-compose pull From 003d7c0e8ca0e794d3f1869e8850820d8541e288 Mon Sep 17 00:00:00 2001 From: Mathieu Lecarme Date: Sun, 3 Apr 2022 13:12:51 +0200 Subject: [PATCH 4/5] Index --- contrib/meilisearch/Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/contrib/meilisearch/Makefile b/contrib/meilisearch/Makefile index cc3fcc3..2ccd796 100644 --- a/contrib/meilisearch/Makefile +++ b/contrib/meilisearch/Makefile @@ -1,13 +1,16 @@ +up: + docker-compose up -d + docker ps + +index: venv + ./venv/bin/python index.py + venv: python3 -m venv venv ./venv/bin/pip install -U pip wheel ./venv/bin/pip install -r requirements.txt -up: - docker-compose up -d - docker ps - down: docker-compose down From 91532b3ca5a74cac42669ae3cf47f596d216013f Mon Sep 17 00:00:00 2001 From: Mathieu Lecarme Date: Sun, 3 Apr 2022 13:13:02 +0200 Subject: [PATCH 5/5] Lis moi. --- contrib/meilisearch/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 contrib/meilisearch/README.md diff --git a/contrib/meilisearch/README.md b/contrib/meilisearch/README.md new file mode 100644 index 0000000..6ac6e66 --- /dev/null +++ b/contrib/meilisearch/README.md @@ -0,0 +1,19 @@ +# Meilisearch + +Indexation naïve et systématique du contenu. + +Fonctionne avec un [meilisearch](https://www.meilisearch.com/) local, ou dans un conteneur, tant que ça écoute http://localhost:7700 + +## Demo + +Lance meilisearch dans un conteneur + + make up + +Index le site (avec du code en python) + + make index + +Mailisearch propose une UI web pour tester la recherche : [http://localhost:7700](http://localhost:7700) + +Il est possible de copier l'index, le dossier `data.ms` sur un serveur, pour effectuer des recherches (en lecture seule).