Commit 44d0a9c6 authored by Jgiannelos's avatar Jgiannelos
Browse files

Merge branch 'reuse-tools' into 'main'

Reuse package. Add OSM changeset verification script

See merge request !17
parents 01f7e794 798084bc
Pipeline #1472 passed with stages
in 2 minutes and 39 seconds
......@@ -28,10 +28,10 @@ build_python:
image: python:latest
script:
- python setup.py sdist
- cp dist/maps-deduped-tilelist*.tar.gz .
- cp dist/wmf-maps-tools*.tar.gz .
artifacts:
paths:
- maps-deduped-tilelist*.tar.gz
- wmf-maps-tools*.tar.gz
tags:
- maps
......@@ -78,7 +78,7 @@ publish_python_package:
artifacts: true
artifacts:
paths:
- maps-deduped-tilelist*.tar.gz
- wmf-maps-tools*.tar.gz
expire_in: never
tags:
- maps
......
# Deduped map tilelist
# WMF maps tools
A collection of maps related CLI utils written in python.
## Dependencies
- Python >= 3.7
- PyYAML
- PyOsmium
- Psycopg 2
## Releases
Project releases are created automatically when a new tag is created. To create a new release with proper packaging:
- Bump the version under `setup.py`
- Bump the version under `debian/changelog`
- Push a new tag with the changes
## Deduped map tilelist
Given an input file with a list of line seperated tiles in `{zoom}/{x}/{y}` format, generate a distinct list of map tiles for all zoom levels from `minzoom` up to `maxzoom`.
## Usage
### Usage
```
usage: maps-deduped-tilelist [-h] minzoom maxzoom [tilelist_file]
......@@ -18,7 +37,7 @@ optional arguments:
-h, --help show this help message and exit
```
## Example
### Example
```
> cat tilelist.txt
......@@ -48,18 +67,24 @@ optional arguments:
1/1/1
```
## Context
### Context
This CLI tool is built as a way to generate a deduplicated tile list of distinct tiles based on the `imposm3` expired tile output. The reason behind this is to deduplicate tiles and optimize tile pregeneration by avoiding generating the same tile multiple times.
## Dependencies
## Imposm changeset verifier
- Python >= 3.7
Given an OSM changeset file and an imposm mapping, verify that operations described in the changeset are reflected in the DB after an OSM sync
## Releases
### Usage
```
usage: imposm-changeset-verify [-h] [--failfast] changeset mapping pgURL
Project releases are created automatically when a new tag is created. To create a new release with proper packaging:
positional arguments:
changeset
mapping
pgURL
- Bump the version under `setup.py`
- Bump the version under `debian/changelog`
- Push a new tag with the changes
optional arguments:
-h, --help show this help message and exit
--failfast
```
\ No newline at end of file
from setuptools import find_packages, setup
setup(
name="maps-deduped-tilelist",
name="wmf-maps-tools",
version="0.0.4",
author="Yiannis Giannelos",
author_email="jgiannelos@wikimedia.org",
description="Generate a distinct list of map tiles up to a zoom level",
description="Collection of WMF maps related python utils",
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Operating System :: OS Independent",
],
package_dir={"": "src"},
packages=find_packages(where="src"),
packages=find_packages(include=["wmfmaps.*"]),
install_requires=["pyyaml", "osmium", "psycopg2"],
python_requires=">=3.7",
entry_points={
"console_scripts": ["maps-deduped-tilelist=tileset.cli:main"],
"console_scripts": [
"maps-deduped-tilelist=wmfmaps.tileset.cli:main",
"imposm-changeset-verify=wmfmaps.changeset.cli:main",
],
},
)
......@@ -2,7 +2,7 @@ from argparse import Namespace
from io import StringIO
from unittest import TestCase, mock
from tileset.cli import main
from wmfmaps.tileset.cli import main
class CliTest(TestCase):
......@@ -16,7 +16,9 @@ class CliTest(TestCase):
test_input_file.write(test_input)
test_input_file.seek(0)
with mock.patch("tileset.cli.argparse.ArgumentParser.parse_args") as parse_args:
with mock.patch(
"wmfmaps.tileset.cli.argparse.ArgumentParser.parse_args"
) as parse_args:
parse_args.return_value = Namespace(
tilelist_file=test_input_file, minzoom=0, maxzoom=1
)
......
from io import StringIO
from unittest import TestCase, mock
from tileset import Tile, TileSet
from wmfmaps.tileset import Tile, TileSet
class TestTile(TestCase):
......@@ -18,8 +18,8 @@ class TestTile(TestCase):
class TestTileSet(TestCase):
@mock.patch("tileset.TileSet.addChildren")
@mock.patch("tileset.TileSet.addParent")
@mock.patch("wmfmaps.tileset.TileSet.addChildren")
@mock.patch("wmfmaps.tileset.TileSet.addParent")
def test_tileset_read(self, mock_addParent, mock_addChildren):
test_input = ["15/5/5", "15/12/12"]
test_input = "\n".join(test_input)
......
import argparse
from wmfmaps.changeset.verifier import Verifier
def main():
parser = argparse.ArgumentParser()
parser.add_argument("changeset")
parser.add_argument("mapping")
parser.add_argument("pgURL")
parser.add_argument("--failfast", action="store_true")
args = parser.parse_args()
verifier = Verifier(args.changeset, args.mapping, args.pgURL, args.failfast)
verifier.verify()
import json
import osmium
import psycopg2
import yaml
class InconsistentChangeException(Exception):
pass
class ElementHandler:
def __init__(self, conn, tables, failfast):
self.conn = conn
self.tables = tables
self.failfast = failfast
self.inconsistencies = []
def exists(self, table, osm_id):
query = f"SELECT * FROM {table} WHERE osm_id = {osm_id}"
cursor = self.conn.cursor()
cursor.execute(query)
return cursor.fetchone() is not None
def verify_deleted(self, change):
for table in self.tables:
if self.exists(table, change.id):
if self.failfast:
raise InconsistentChangeException(
f"Element with id:{change.id} shouldn't exist"
)
self.inconsistencies.append(
{"type": "should be deleted", "id": change.id}
)
break
def verify_existing(self, change):
at_least_once = False
for table in self.tables:
if self.exists(table, change.id):
at_least_once = True
break
if not at_least_once:
if self.failfast:
raise InconsistentChangeException(
f"Element with id:{change.id} should exist"
)
self.inconsistencies.append({"type": "should exist", "id": change.id})
def log_inconsistencies(self):
for elem in self.inconsistencies:
print(f"{json.dumps(elem)}")
def __call__(self, change):
if change.deleted:
self.verify_deleted(change)
else:
self.verify_existing(change)
class Verifier:
def __init__(self, changesetPath, mappingPath, pgURL, failfast=True):
self.changesetPath = changesetPath
self.mappingPath = mappingPath
self.pgURL = pgURL
self.tables = self.get_tables()
self.conn = psycopg2.connect(self.pgURL)
self.conn.set_session(readonly=True)
self.failfast = failfast
def get_tables(self):
with open(self.mappingPath, "r") as f:
mapping = yaml.safe_load(f)
return mapping["tables"].keys()
def verify(self):
node = ElementHandler(self.conn, self.tables, self.failfast)
way = ElementHandler(self.conn, self.tables, self.failfast)
relation = ElementHandler(self.conn, self.tables, self.failfast)
handler = osmium.make_simple_handler(node=node, way=way, relation=relation)
handler.apply_file(self.changesetPath)
node.log_inconsistencies()
way.log_inconsistencies()
relation.log_inconsistencies()
import argparse
import sys
from tileset import TileSet
from wmfmaps.tileset import TileSet
def main():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment