Commit ce262309 authored by Jgiannelos's avatar Jgiannelos
Browse files

Add script for OSM changeset verification

parent 543de9e2
Pipeline #1468 passed with stages
in 2 minutes and 23 seconds
......@@ -12,8 +12,12 @@ setup(
"Operating System :: OS Independent",
],
packages=find_packages(include=["wmfmaps.*"]),
install_requires=["pyyaml", "osmium", "psycopg2"],
python_requires=">=3.7",
entry_points={
"console_scripts": ["maps-deduped-tilelist=wmfmaps.tileset.cli:main"],
"console_scripts": [
"maps-deduped-tilelist=wmfmaps.tileset.cli:main",
"imposm-changeset-verify=wmfmaps.changeset.cli:main",
],
},
)
import argparse
from wmfmaps.changeset.verifier import Verifier
def main():
parser = argparse.ArgumentParser()
parser.add_argument("changeset")
parser.add_argument("mapping")
parser.add_argument("pgURL")
parser.add_argument("--failfast", action="store_true")
args = parser.parse_args()
verifier = Verifier(args.changeset, args.mapping, args.pgURL, args.failfast)
verifier.verify()
import osmium
import psycopg2
import yaml
class InconsistentChangeException(Exception):
pass
class ElementHandler:
def __init__(self, conn, tables, failfast):
self.conn = conn
self.tables = tables
self.failfast = failfast
self.inconsistencies = []
def exists(self, table, osm_id):
query = f"SELECT * FROM {table} WHERE osm_id = {osm_id}"
cursor = self.conn.cursor()
cursor.execute(query)
return cursor.fetchone() is not None
def verify_deleted(self, change):
for table in self.tables:
if self.exists(table, change.id):
if self.failfast:
raise InconsistentChangeException(
f"Element with id:{change.id} shouldn't exist"
)
self.inconsistencies.append(change.id)
def verify_existing(self, change):
for table in self.tables:
if not self.exists(table, change.id):
if self.failfast:
raise InconsistentChangeException(
f"Element with id:{change.id} should exist"
)
self.inconsistencies.append(change.id)
def log_inconsistencies(self):
for elem in self.inconsistencies:
print(f"Inconsistency with OSM ID: {elem}")
def __call__(self, change):
if change.deleted:
self.verify_deleted(change)
else:
self.verify_existing(change)
class Verifier:
def __init__(self, changesetPath, mappingPath, pgURL, failfast=True):
self.changesetPath = changesetPath
self.mappingPath = mappingPath
self.pgURL = pgURL
self.tables = self.get_tables()
self.conn = psycopg2.connect(self.pgURL)
self.conn.set_session(readonly=True)
self.failfast = failfast
def get_tables(self):
with open(self.mappingPath, "r") as f:
mapping = yaml.safe_load(f)
return mapping["tables"].keys()
def verify(self):
node = ElementHandler(self.conn, self.tables, self.failfast)
way = ElementHandler(self.conn, self.tables, self.failfast)
relation = ElementHandler(self.conn, self.tables, self.failfast)
handler = osmium.make_simple_handler(node=node, way=way, relation=relation)
handler.apply_file(self.changesetPath)
node.log_inconsistencies()
way.log_inconsistencies()
relation.log_inconsistencies()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment