From a05f4fff324f512840b14a47f9e5ad94aba8c2c0 Mon Sep 17 00:00:00 2001 From: TheFGFSEagle Date: Fri, 29 Jul 2022 15:40:14 +0200 Subject: [PATCH] Added shapefile and DEM processing (for TerraGear) and installer scripts --- install.py | 66 ++++++++++++++ scenery/process-elevations.py | 40 +++++++++ scenery/process-shapefiles.py | 163 ++++++++++++++++++++++++++++++++++ scenery/shapefiles.py | 9 ++ utils/constants.py | 12 +++ 5 files changed, 290 insertions(+) create mode 100755 install.py create mode 100755 scenery/process-elevations.py create mode 100755 scenery/process-shapefiles.py create mode 100755 scenery/shapefiles.py create mode 100755 utils/constants.py diff --git a/install.py b/install.py new file mode 100755 index 0000000..4621bf4 --- /dev/null +++ b/install.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +#-*- coding:utf-8 -*- + +# Installer script for the main scripts and modules of fgtools +import os +import sys +import argparse +import shutil +import site + +from fgtools.utils import constants + +argp = argparse.ArgumentParser(description="install.py - installs the TerraGear tools so that they can be run like any other executable") + +argp.add_argument( + "-p", "--prefix", + help="Installation prefix (default: %(default)s)", + default=os.environ.get("FGTOOLSPREFIX", os.path.join(constants.HOME, ".local"))) +) + +argp.add_argument( + "--add-to-path", + help="whether to modify your $HOME/.profile file to have the folder containing the scripts in your path (default: %(default)s", + default="yes", + choices=["yes", "no"] +) + +args = argp.parse_args() + +SCRIPTDIR = os.path.dirname(os.path.abspath(__name__)) +BINDIR = os.path.join(args.prefix, "bin") +PYLIBDIR = os.path.join(args.prefix, "lib", "python3") + +print(f"Installing to {args.prefix}") + +print("Installing modules …") +shutil.copytree(os.path.join(SCRIPTDIR, constants.MODULE), PYLIBDIR) + +print("Installing scripts …") +for script in constants.SCRIPTS: + shutil.copy2(os.path.join(SCRIPTDIR, script), BINDIR) + +if not BINDIR in os.environ.get("PATH", "").split(os.pathsep): + if args.add_to_path: + print(f"Adding {BINDIR} to your $PATH …") + + if os.name == "posix": # Linux, MacOS + with open(os.path.join(constants.HOME, ".profile"), "a") as f: + f.write(f"export PATH=\"$PATH{os.pathsep}{BINDIR}\"") + else: # probably Windows + os.system(f"setx PATH \"%PATH%{os.pathsep}{BINDIR}\"") + else: + print(f"WARNING: {BINDIR} was not added to your $PATH - please do that manually") + +if not LIBDIR in os.environ.get("PYTHONPATH", "").split(os.pathsep): + print(f"Adding {LIBDIR} to your $PYTHONPATH …") + + if not os.path.isdir(site.USER_SITE): + os.mkdirs(site.USER_SITE) + + mode = "a" + if not os.path.isfile(os.path.join(site.USER_SITE, "sitecustomize.py")): + mode = "w" + + with open(os.path.join(site.USER_SITE, "sitecustomize.py"), mode) as f: + f.write(f"import site; site.addsitedir({LIBDIR})") diff --git a/scenery/process-elevations.py b/scenery/process-elevations.py new file mode 100755 index 0000000..0e8860c --- /dev/null +++ b/scenery/process-elevations.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- + +# Small wrapper script for gdalchop that, if you are following the standar TerraGear directory structure, +# can be run without arguments + +import sys +import argparse +import subprocess + +from tgtools import constants + +argp = argparse.ArgumentParser(description="process-elevations.py - process a directory of elevation data files with gdalchop and terrafit") + +argp.add_argument( + "-i", "--input-folder", + help="folder containing the raw elevation files (default: %(default)s)", + default="./data/elevation", + metavar="INPUT", +) + +argp.add_argument( + "-o", "--output-folder", + help="where to put the produced files (default: %(default)s)", + default="./work/elevation", + metavar="OUTPUT" +) + +argp.add_argument( + "-v", "--version", + action="version", + version=f"FGTools {constants.__versionstr__}" +) + +args = argp.parse_args() + +chop = subprocess.Popen(["gdalchop", args.output_folder, args.input_folder], stdout=sys.stdout, stderr=sys.stderr) +exit = chop.run() +sys.exit(exit) + diff --git a/scenery/process-shapefiles.py b/scenery/process-shapefiles.py new file mode 100755 index 0000000..ae3f245 --- /dev/null +++ b/scenery/process-shapefiles.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +#-*- coding:utf-8 -*- + +# Script that merges, slices, and decodes OSM shapefiles as needed by parsing the coordinates input and the extent of all shapefiles. + +import os +import sys +import glob +import re +import argparse +import subprocess + +DESCRIPTION = """ +process-shapefiles.py - merges, slices, and decodes OSM shapefiles + +IMPORTANT: CORINE shapefiles are NOT YET SUPPORTED !!! + +This script recursively searches the specified input directory for files containing 'osm' and ending with '.shp'. +All that are found are then categorized into multiple categories - one for landuse, one for landmass, one for roads, etc. +For this reason, you may only remove the 'gis_' and '_free_1' parts from the shapefile's names ! +Everything else in the name must be conserved in order for your resulting scenery not to have big areas of default landmass terrain !'' + +Then, for each shapefile in each category the extents will be queried using ogrinfo. To reduce processing time on subsequent runs, the results will be cached. +Then, the script will decide whether to merge or slice the shapefiles for each category based on the coordinates you input. +It will merge / slice the files accordingly with ogr2ogr. +As the final step, the resulting shapefiles will be decoded into files that tg-constrcut can read using ogr-decode. +""" + +def find(src): + osm_shapefiles = [] + shapefiles = glob.glob(os.path.join(src, "**", "**.shp") + for file in files: + if "osm" in os.path.split(file): + osm_shapefiles.append(file) + return sorted(osm_shapefiles) + +def categorize(shapefiles): + catnames = ["buildings", "landuse", "natural", "places", "pofw", "pois", "railways", "roads", "traffic", "transport", "water", "waterways"] + categorized = [] + + for shapefile in shapefiles: + name = os.path.split(shapefile)[-1].split(".")[0] + + # Skip if the name is of the form gis_osm_landuse_a_free_1 + # these files are much smaller than the ones without _a_, probably contain less data. + if "_a_" in name: + continue + + name = re.sub(r"gis|osm|a|free|_|(1-9)", "", name) # gis_osm_landuse_a_free_1 becomes just landuse + # Skip if the name is not a recognized catname + if not name in catnames: + continue + + categorized.append({"path": shapefile, "category": name}) + return categorized + +def get_extents(categorized): + for shapefile in categorized: + cmd = f"ogrinfo -al -so -ro -nocount -nomd {shapefile['path']}" + query = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = list(map(lambda s: s.decode(), query.stdout.splitlines())) # subprocess.Popen.stdout is a binary file - we need normal strings + + extents = [s for s in output if "Extent" in s] + feature_count = [s for s in output if "Feature Count" in s] + + if len(extents) != 1 or len(feature_count) != 1: + print("ERROR: Fetching shapefile information using ogrinfo failed.") + print(" Try reinstalling it through your package manager.") + print(" If that doesn't help, please file a bug report at .") + print(" If you do that, please attach the process-shapefiles-bugreport.md file in order for the maintainers to be able to help you.") + + with open("process-shapefile-bugreport.md", "w") as f: + f.write(f"### Output of `{cmd}`") + f.writelines(output) + f.write(f"### ogrinfo version:") + f.write(subprocess.run("ogrinfo --version", stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True).stdout.deocde()) + sys.exit(2) + + # convert "Extent: (10.544425, 51.500000) - (11.500000, 52.500000)" to {"xll": 10.544425, "yll": 51.5, "xur": 11.5, "yur": 52,5] + extents = dict(zip(["xll", "yll", "yur", "yur"], map(float, re.sub(r"Extent:\s\(|\)", "", extents[0]).replace(") - (", ", ").split(", ")))) + shapefile["extents"] = extents + + +def merge_slice(shapefiles, coords): + pass + +def decode(shapefiles, dest): + pass + +if __name__ == "__main__": + argp = argparse.ArgumentParser(description="process-shapefiles.py - merges, slices, and decodes OSM shapefiles") + + argp.add_argument( + "-v", "--version", + action="version", + version=f"TerraGear tools {'.'.join(map(str, constants.__version__))}" + ) + + argp.add_argument( + "-d", "--description", + help="display an extended description of what this script does and exit" + ) + + argp.add_argument( + "-i", "--input-folder", + help="folder containing folder 'shapefiles_raw' containing folders containing unprocessed shapefiles(default: %(default)s)", + default="./data", + metavar="FOLDER" + ) + + argp.add_argument( + "-o", "--output-folder", + help="folder to put ogr-decode result into (default: %(default)s)", + default="./work", + metavar="FOLDER" + ) + + argp.add_argument( + "-c", "--cache-folder", + help="where to put cache folder (default: %(default)s)", + default=os.path.join(constants.HOME, ".cache", "tgtools") + ) + + argp.add_argument( + "-l", "--lower-left", + help="coordinates of the lower left corner of the bounding box of the region that shapefiles should be processed for (default: %(default)s)", + default="-180,-90" + ) + + argp.add_argument( + "-u", "--upper-right", + help="coordinates of the upper-right corner of the bounding box of the region that shapefiles should be processed for (default: %(default)s)", + default="180,90" + ) + + args = argp.parse_args() + + if args.description: + print(DESCRIPTION) + sys.exit(0) + + src = args.input_folder + dest = args.output_folder + cache = args.cache_folder + xll, yll = args.lower_left.split(",") + xur, yur = args.upper_right.split(",") + coords = {"xll": xll, "yll": yll, "xur": xur, "yur": yur} + + if not os.path.isdir(src): + print(f"ERROR: input folder {args.input_folder} does not exist, exiting") + sys.exit(1) + + if not os.path.isdir(dst): + os.mkdirs(dst) + + if not os.path.isdir(cache): + os.mkdirs(cache) + + shapefiles = shapefiles.find(src) + categories = shapefiles.categorize(shapefiles) + extents = shapefiles.get_extents(categorized) + shapefiles = shapefiles.merge_slice(extents, coords) + result = shapefiles.decode(shapefiles, dest) diff --git a/scenery/shapefiles.py b/scenery/shapefiles.py new file mode 100755 index 0000000..d4a92ef --- /dev/null +++ b/scenery/shapefiles.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +#-*- coding:utf-8 -*- + +import os +import sys +import glob +import re + + diff --git a/utils/constants.py b/utils/constants.py new file mode 100755 index 0000000..63228d5 --- /dev/null +++ b/utils/constants.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +#-*- coding:utf-8 -*- + +import os +import sys + +HOME = os.environ.get("HOME", os.path.expanduser("~")) +SCRIPTS = ["process-shapefiles.py", "process-elevations.py"] +MODULE = "tgtools" +__version__ = (1, 0, 0) +__versionstr__ = ".".join(map(str, __version__)) +