Merge branch 'reuse-tools' into 'main'

Reuse package. Add OSM changeset verification script

See merge request !17

.gitlab-ci.yml

@@ -28,10 +28,10 @@ build_python:
   image: python:latest
   script:
     - python setup.py sdist
-    - cp dist/maps-deduped-tilelist*.tar.gz .
+    - cp dist/wmf-maps-tools*.tar.gz .
   artifacts:
     paths:
-      - maps-deduped-tilelist*.tar.gz
+      - wmf-maps-tools*.tar.gz
   tags:
     - maps
 
@@ -78,7 +78,7 @@ publish_python_package:
       artifacts: true
   artifacts:
     paths:
-      - maps-deduped-tilelist*.tar.gz
+      - wmf-maps-tools*.tar.gz
     expire_in: never
   tags:
     - maps

README.md

-# Deduped map tilelist
+# WMF maps tools
+
+A collection of maps related CLI utils written in python.
+
+## Dependencies
+
+- Python >= 3.7
+- PyYAML
+- PyOsmium
+- Psycopg 2
+
+## Releases
+
+Project releases are created automatically when a new tag is created. To create a new release with proper packaging:
+
+- Bump the version under `setup.py`
+- Bump the version under `debian/changelog`
+- Push a new tag with the changes
+
+## Deduped map tilelist
 
 Given an input file with a list of line seperated tiles in `{zoom}/{x}/{y}` format, generate a distinct list of map tiles for all zoom levels from `minzoom` up to `maxzoom`.
 
-## Usage
+### Usage
 
 ```
 usage: maps-deduped-tilelist [-h] minzoom maxzoom [tilelist_file]
@@ -18,7 +37,7 @@ optional arguments:
   -h, --help     show this help message and exit
 ```
 
-## Example
+### Example
 
 ```
 > cat tilelist.txt
@@ -48,18 +67,24 @@ optional arguments:
 1/1/1
 ```
 
-## Context
+### Context
 
 This CLI tool is built as a way to generate a deduplicated tile list of distinct tiles based on the `imposm3` expired tile output. The reason behind this is to deduplicate tiles and optimize tile pregeneration by avoiding generating the same tile multiple times.
 
-## Dependencies
+## Imposm changeset verifier
 
-- Python >= 3.7
+Given an OSM changeset file and an imposm mapping, verify that operations described in the changeset are reflected in the DB after an OSM sync
 
-## Releases
+### Usage
+```
+usage: imposm-changeset-verify [-h] [--failfast] changeset mapping pgURL
 
-Project releases are created automatically when a new tag is created. To create a new release with proper packaging:
+positional arguments:
+  changeset
+  mapping
+  pgURL
 
-- Bump the version under `setup.py`
-- Bump the version under `debian/changelog`
-- Push a new tag with the changes
+optional arguments:
+  -h, --help  show this help message and exit
+  --failfast
+```
\ No newline at end of file

setup.py

 from setuptools import find_packages, setup
 
 setup(
-    name="maps-deduped-tilelist",
+    name="wmf-maps-tools",
     version="0.0.4",
     author="Yiannis Giannelos",
     author_email="jgiannelos@wikimedia.org",
-    description="Generate a distinct list of map tiles up to a zoom level",
+    description="Collection of WMF maps related python utils",
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
         "Operating System :: OS Independent",
     ],
-    package_dir={"": "src"},
-    packages=find_packages(where="src"),
+    packages=find_packages(include=["wmfmaps.*"]),
+    install_requires=["pyyaml", "osmium", "psycopg2"],
     python_requires=">=3.7",
     entry_points={
-        "console_scripts": ["maps-deduped-tilelist=tileset.cli:main"],
+        "console_scripts": [
+            "maps-deduped-tilelist=wmfmaps.tileset.cli:main",
+            "imposm-changeset-verify=wmfmaps.changeset.cli:main",
+        ],
     },
 )

tests/test_cli.py

@@ -2,7 +2,7 @@ from argparse import Namespace
 from io import StringIO
 from unittest import TestCase, mock
 
-from tileset.cli import main
+from wmfmaps.tileset.cli import main
 
 
 class CliTest(TestCase):
@@ -16,7 +16,9 @@ class CliTest(TestCase):
         test_input_file.write(test_input)
         test_input_file.seek(0)
 
-        with mock.patch("tileset.cli.argparse.ArgumentParser.parse_args") as parse_args:
+        with mock.patch(
+            "wmfmaps.tileset.cli.argparse.ArgumentParser.parse_args"
+        ) as parse_args:
             parse_args.return_value = Namespace(
                 tilelist_file=test_input_file, minzoom=0, maxzoom=1
             )

tests/test_tileset.py

 from io import StringIO
 from unittest import TestCase, mock
 
-from tileset import Tile, TileSet
+from wmfmaps.tileset import Tile, TileSet
 
 
 class TestTile(TestCase):
@@ -18,8 +18,8 @@ class TestTile(TestCase):
 
 
 class TestTileSet(TestCase):
-    @mock.patch("tileset.TileSet.addChildren")
-    @mock.patch("tileset.TileSet.addParent")
+    @mock.patch("wmfmaps.tileset.TileSet.addChildren")
+    @mock.patch("wmfmaps.tileset.TileSet.addParent")
     def test_tileset_read(self, mock_addParent, mock_addChildren):
         test_input = ["15/5/5", "15/12/12"]
         test_input = "\n".join(test_input)

wmfmaps/changeset/cli.py

+import argparse
+
+from wmfmaps.changeset.verifier import Verifier
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("changeset")
+    parser.add_argument("mapping")
+    parser.add_argument("pgURL")
+    parser.add_argument("--failfast", action="store_true")
+    args = parser.parse_args()
+
+    verifier = Verifier(args.changeset, args.mapping, args.pgURL, args.failfast)
+    verifier.verify()

wmfmaps/changeset/verifier.py

+import json
+import osmium
+import psycopg2
+import yaml
+
+
+class InconsistentChangeException(Exception):
+    pass
+
+
+class ElementHandler:
+    def __init__(self, conn, tables, failfast):
+        self.conn = conn
+        self.tables = tables
+        self.failfast = failfast
+        self.inconsistencies = []
+
+    def exists(self, table, osm_id):
+        query = f"SELECT * FROM {table} WHERE osm_id = {osm_id}"
+        cursor = self.conn.cursor()
+        cursor.execute(query)
+        return cursor.fetchone() is not None
+
+    def verify_deleted(self, change):
+        for table in self.tables:
+            if self.exists(table, change.id):
+                if self.failfast:
+                    raise InconsistentChangeException(
+                        f"Element with id:{change.id} shouldn't exist"
+                    )
+                self.inconsistencies.append(
+                    {"type": "should be deleted", "id": change.id}
+                )
+                break
+
+    def verify_existing(self, change):
+        at_least_once = False
+        for table in self.tables:
+            if self.exists(table, change.id):
+                at_least_once = True
+                break
+
+        if not at_least_once:
+            if self.failfast:
+                raise InconsistentChangeException(
+                    f"Element with id:{change.id} should exist"
+                )
+            self.inconsistencies.append({"type": "should exist", "id": change.id})
+
+    def log_inconsistencies(self):
+        for elem in self.inconsistencies:
+            print(f"{json.dumps(elem)}")
+
+    def __call__(self, change):
+        if change.deleted:
+            self.verify_deleted(change)
+        else:
+            self.verify_existing(change)
+
+
+class Verifier:
+    def __init__(self, changesetPath, mappingPath, pgURL, failfast=True):
+        self.changesetPath = changesetPath
+        self.mappingPath = mappingPath
+        self.pgURL = pgURL
+        self.tables = self.get_tables()
+        self.conn = psycopg2.connect(self.pgURL)
+        self.conn.set_session(readonly=True)
+        self.failfast = failfast
+
+    def get_tables(self):
+        with open(self.mappingPath, "r") as f:
+            mapping = yaml.safe_load(f)
+            return mapping["tables"].keys()
+
+    def verify(self):
+        node = ElementHandler(self.conn, self.tables, self.failfast)
+        way = ElementHandler(self.conn, self.tables, self.failfast)
+        relation = ElementHandler(self.conn, self.tables, self.failfast)
+        handler = osmium.make_simple_handler(node=node, way=way, relation=relation)
+        handler.apply_file(self.changesetPath)
+
+        node.log_inconsistencies()
+        way.log_inconsistencies()
+        relation.log_inconsistencies()

src/tileset/cli.py → wmfmaps/tileset/cli.py

 import argparse
 import sys
 
-from tileset import TileSet
+from wmfmaps.tileset import TileSet
 
 
 def main():