From c80a2dd40e4e5c03dc411e3a6578b77d0c3f67c8 Mon Sep 17 00:00:00 2001 From: TheFGFSEagle Date: Fri, 4 Nov 2022 21:42:51 +0100 Subject: [PATCH] Added scripts to scrape PDF's, make table operations, convert coordinates between different formats --- aircraft/vsphist2jsbtable.py | 123 ++++++++++++++++++++++++-- fgtools/utils/__init__.py | 13 +++ fgtools/utils/interpolator.py | 10 ++- misc/scrape-emanualonline-pdf.py | 63 +++++++++++++ misc/scrape-scribd.py | 112 ++++++++++++++++++++++++ misc/tabletool.py | 146 +++++++++++++++++++++++++++++++ scenery/aptdat2airportsxml.py | 5 +- scenery/coord_convert.py | 46 ++++++++++ 8 files changed, 504 insertions(+), 14 deletions(-) create mode 100644 misc/scrape-emanualonline-pdf.py create mode 100644 misc/scrape-scribd.py create mode 100644 misc/tabletool.py create mode 100644 scenery/coord_convert.py diff --git a/aircraft/vsphist2jsbtable.py b/aircraft/vsphist2jsbtable.py index b169cfb..ef2f128 100755 --- a/aircraft/vsphist2jsbtable.py +++ b/aircraft/vsphist2jsbtable.py @@ -3,7 +3,12 @@ import argparse import os, sys +import math + +import scipy.interpolate + from fgtools.utils.interpolator import Interpolator +from fgtools.utils import range class Case: def __init__(self, textcase): @@ -25,7 +30,12 @@ class Case: textcase.pop(0) # get rid of the Solver case line textcase.pop(0) # get rid of another legend line lastiterindex = textcase.index("Skin Friction Drag Break Out:") - 1 - self.CL, self.CDo, self.CDi, self.CDtot, self.CS, self.LD, self.E, self.CFx, self.CFy, self.CFz, self.CMx, self.CMy, self.CMz, self.CDtrefftz, self.TQS = map(float, textcase.pop(lastiterindex).split()[4:]) # we don't need Mach, AoA, Beta again + line = textcase.pop(lastiterindex).split()[4:] + if len(line) == 16: + self.CL, self.CDo, self.CDi, self.CDtot, self.CDt, self.CDtot_t, self.CS, self.LD, self.E, self.CFx, self.CFy, self.CFz, self.CMx, self.CMy, self.CMz, self.TQS = map(float, line) + else: + self.CL, self.CDo, self.CDi, self.CDtot, self.CS, self.LD, self.E, self.CFx, self.CFy, self.CFz, self.CMx, self.CMy, self.CMz, self.CDtrefftz, self.TQS = map(float, line) + def __str__(self): return "Case(" + ", ".join(str(k) + " = " + str(v) for k, v in vars(self).items()) + ")" @@ -33,7 +43,7 @@ class Case: def __repr__(self): return self.__str__() -def get_cases(path): +def get_cases(path, mach): path = os.path.abspath(path) if not path.endswith(".history"): @@ -56,13 +66,21 @@ def get_cases(path): for textcase in textcases: case = Case(textcase) + if case.RollRate or case.YawRate or case.PitchRate or case.Mach != mach: + continue if not case.AoA in cases: cases[case.AoA] = {} cases[case.AoA][case.Beta] = case + mostbetas = len(max(cases.values(), key=lambda d: len(d))) + for AoA in list(cases.keys()): + if len(cases[AoA]) < mostbetas: + del cases[AoA] + continue + return cases -def print_table(cases, coeff, indent, precision): +def get_raw_coeffs(cases, coeff): coeffs = {} for AoA in cases: for Beta in cases[AoA]: @@ -73,13 +91,37 @@ def print_table(cases, coeff, indent, precision): if not AoA in coeffs: coeffs[AoA] = {} coeffs[AoA][Beta] = getattr(cases[AoA][Beta], coeff) + return coeffs + +def get_interpolated_coeffs(cases, coeff, alphas, betas, symmetrize): + coeffs = get_raw_coeffs(cases, coeff) + ralphas = list(coeffs.keys()) + rbetas = list(coeffs[ralphas[0]].keys()) + values = list([list(coeffs[ralpha].values()) for ralpha in ralphas]) + interp = scipy.interpolate.RectBivariateSpline(ralphas, rbetas, values) + #interp = scipy.interpolate.interp2d(rbetas, ralphas, values, fill_value=None) + coeffs = {} + for i, alpha in enumerate(alphas): + if not alpha in coeffs: + coeffs[alpha] = {} + for j, beta in enumerate(betas): + if symmetrize: + s = interp(alpha, beta, grid=False) + coeffs[alpha][beta] = math.copysign((abs(interp(alpha, beta, grid=False)) + abs(interp(alpha, -beta, grid=False))) / 2, s) + else: + coeffs[alpha][beta] = interp(alpha, beta, grid=False) + #coeffs[alpha][beta] = interp([beta], [alpha]) + return coeffs +def print_table(coeffs, indent, precision, use_wing_alpha): print("") - print(indent + 'aero/alpha-deg') - print(indent + 'aero/beta-deg') + print(indent + 'aero/alpha-' + ("wing-" if use_wing_alpha else "") + 'deg') + if len(coeffs[list(coeffs.keys())[0]]) > 1: + print(indent + 'aero/beta-deg') #print(indent + 'velocities/mach') print(indent + "") - print(indent + indent + indent + indent + (indent + indent).join(map(str, coeffs[list(coeffs.keys())[0]].keys()))) + if len(coeffs[list(coeffs.keys())[0]]) > 1: + print(indent + indent + indent + indent + (indent + indent).join(map(str, coeffs[list(coeffs.keys())[0]].keys()))) for AoA in coeffs: print(indent + indent + str(AoA), end="") for Beta in coeffs[AoA]: @@ -111,13 +153,78 @@ if __name__ == "__main__": default=6 ) + argp.add_argument( + "--alpha-min", + help="Lowest alpha table lookup value", + type=float, + default=None + ) + argp.add_argument( + "--alpha-max", + help="Highest alpha table lookup value", + type=float, + default=None + ) + argp.add_argument( + "--alpha-step", + help="Alpha table lookup value step size", + type=float, + default=None + ) + + argp.add_argument( + "--beta-min", + help="Lowest alpha table lookup value", + type=float, + default=None + ) + argp.add_argument( + "--beta-max", + help="Highest alpha table lookup value", + type=float, + default=None + ) + argp.add_argument( + "--beta-step", + help="Alpha table lookup value step size", + type=float, + default=None + ) + + argp.add_argument( + "-s", "--symmetrize", + help="Symmetrize table around the sideslip axis", + action="store_true", + ) + + argp.add_argument( + "-w", "--use-wing-alpha", + help="Use aero/alpha-wing-deg instead of aero/alpha-deg for the alpha lookup property", + action="store_true" + ) + argp.add_argument( "input_file", help="VSPAERO .history file", ) + argp.add_argument( + "-m", "--mach", + help="Mach for which to output table - must exist in the .history file !", + type=float, + required=True + ) + args = argp.parse_args() - cases = get_cases(args.input_file) - print_table(cases, args.coeff, args.indentation, args.precision) + cases = get_cases(args.input_file, args.mach) + if None not in (args.alpha_min, args.alpha_max, args.alpha_step, args.beta_min, args.beta_max, args.beta_step): + alphas = list(range(args.alpha_min, args.alpha_max, args.alpha_step)) + alphas.append(args.alpha_max) + betas = list(range(args.beta_min, args.beta_max, args.beta_step)) + betas.append(args.beta_max) + coeffs = get_interpolated_coeffs(cases, args.coeff, alphas, betas, args.symmetrize) + else: + coeffs = get_raw_coeffs(cases, args.coeff) + print_table(coeffs, args.indentation, args.precision, args.use_wing_alpha) diff --git a/fgtools/utils/__init__.py b/fgtools/utils/__init__.py index 5f9127f..cb7639e 100644 --- a/fgtools/utils/__init__.py +++ b/fgtools/utils/__init__.py @@ -125,3 +125,16 @@ def wrap_period(n, min, max): return n +def range(stop, start=None, step=1): + if start: + stop, start = start, stop + else: + start = 0 + yield round(start, 14) + i = 0 + r = start + while r < stop: + i += 1 + r = start + i * step + yield round(r, 14) + diff --git a/fgtools/utils/interpolator.py b/fgtools/utils/interpolator.py index dd8fa39..57e8855 100644 --- a/fgtools/utils/interpolator.py +++ b/fgtools/utils/interpolator.py @@ -33,9 +33,13 @@ class Interpolator: raise ValueError(f"Interpolator.interpolate: cannot interpolate on a table with less than two data points") # only sort if not already sorted to increase performance for large tables - if not self._sorted and sort: - self._indexes.sort() - self._values.sort() + if sort and not self._sorted: + items = sorted(list(zip(self._indexes, self._values)), key=lambda t: t[0]) + self._indexes = [] + self._values = [] + for item in items: + self._indexes.append(item[0]) + self._values.append(item[1]) self._sorted = True if index in self._indexes: diff --git a/misc/scrape-emanualonline-pdf.py b/misc/scrape-emanualonline-pdf.py new file mode 100644 index 0000000..d38e7a9 --- /dev/null +++ b/misc/scrape-emanualonline-pdf.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +import os +import argparse +import re +import requests + +from fgtools.utils import constants + +pattern = r'(?<=src=")https:\/\/static-repo.emanualonline.com\/.+\.jpg(?=")' + +def download_pages(url, output): + html = requests.get(url).text + urls = re.findall(pattern, html) + urltemplate = "/".join(urls[0].split("/")[:-2] + ["%d", "%d.jpg"]) + + paths = [] + i = 1 + while True: + page = requests.get(urltemplate % (i, i)) + i += 1 + if page.status_code != 200: + break + + path = os.path.join(constants.CACHEDIR, os.path.split(output)[-1] + f"-{i}.jpg") + paths.append(path) + with open(path, "wb") as f: + f.write(page.content) + + return paths + +def write_pdf(paths, output): + print(f"Joining {len(paths)} JPG files into {output} … ", end="") + newpaths = " ".join([f'"{path}"' for path in paths]) + os.system(f'img2pdf {newpaths} --output "{output}"') + print("done.") + print("Deleting JPG files … ", end="") + for path in paths: + os.remove(path) + print("done") + +if __name__ == "__main__": + argp = argparse.ArgumentParser() + + argp.add_argument( + "url", + help="URL to emanualonline.com PDF offer" + ) + + argp.add_argument( + "-o", "--output", + help="Output file", + required=True + ) + + args = argp.parse_args() + + os.makedirs(os.path.join(*os.path.split(os.path.relpath(args.output))[:-1]) or ".", exist_ok=True) + + paths = download_pages(args.url, args.output) + write_pdf(paths, args.output) + diff --git a/misc/scrape-scribd.py b/misc/scrape-scribd.py new file mode 100644 index 0000000..d643c67 --- /dev/null +++ b/misc/scrape-scribd.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +import os +import argparse +import re +import requests + +from PIL import Image + +from bs4 import BeautifulSoup + +from fgtools.utils import constants +json_pattern = r'(?<=content-url: ")https:\/\/html.scribdassets.com\/.+\.jsonp(?=")' +img_pattern = r'' + +class JSPage: + def __init__(self, number, width, height, url): + self.number = number + self.width = width + self.height = height + self.url = url + + def get_image(self): + text = requests.get(self.url).text + images = list(map(lambda s: BeautifulSoup(s.replace("\\", ""), features="lxml").body.find("img"), re.findall(img_pattern, text))) + print(len(images)) + src_image = Image.open(requests.get(images[0]["orig"], stream=True).raw) + pil_image = Image.new("RGB", (self.width, self.height)) + pil_image.paste((255, 255, 255), (0, 0, pil_image.size[0], pil_image.size[1])) + for image in images: + style = {} + for item in image["style"].split(";"): + item = item.split(":") + style[item[0]] = item[1].replace("px", "") + + clip = style["clip"] + clip = {k: int(v) for k, v in zip(("top", "right", "bottom", "left"), clip[clip.find("(") + 1:-1].split(" "))} + cropped_src_image = src_image.copy().crop((clip["left"], clip["top"], clip["right"], clip["bottom"])) + pil_image.paste(cropped_src_image, (int(style["left"]) + clip["left"], int(style["top"]) + clip["top"])) + + return pil_image + +def parse_pages_script(script): + lines = list(map(str.strip, script.split("\n")[1:-1])) + number = 0 + width = 0 + height = 0 + url = "" + pages = [] + for line in lines: + if "pageNum" in line: + number = int(line.split(": ")[1][:-1]) + elif "origWidth" in line: + width = int(line.split(": ")[1][:-1]) + elif "origHeight" in line: + height = int(line.split(": ")[1][:-1]) + elif "contentUrl" in line: + url = line.split(": ")[1][1:-1] + + if number and width and height and url: + page = JSPage(number, width, height, url) + pages.append(page) + number = width = height = 0 + url = "" + + return pages + +def download_pages(url, output): + html = BeautifulSoup(requests.get(url).text, features="lxml") + pages_script = html.body.find("div", attrs={"class": "outer_page_container"}).find("script", attrs={"type": "text/javascript"}) + pages = sorted(parse_pages_script(str(pages_script)), key=lambda p: p.number) + + paths = [] + for page in pages: + path = os.path.join(constants.CACHEDIR, os.path.split(output)[-1] + f"-{page.number}.jpg") + paths.append(path) + page.get_image().save(path, "JPEG") + + return paths + +def write_pdf(paths, output): + print(f"Joining {len(paths)} JPG files into {output} … ", end="") + newpaths = " ".join([f'"{path}"' for path in paths]) + os.system(f'img2pdf {newpaths} --output "{output}"') + print("done.") + print("Deleting JPG files … ", end="") + for path in paths: + os.remove(path) + print("done") + +if __name__ == "__main__": + argp = argparse.ArgumentParser() + + argp.add_argument( + "url", + help="URL to Scribd web page" + ) + + argp.add_argument( + "-o", "--output", + help="Output file", + required=True + ) + + args = argp.parse_args() + + os.makedirs(os.path.join(*os.path.split(os.path.relpath(args.output))[:-1]) or ".", exist_ok=True) + + paths = download_pages(args.url, args.output) + write_pdf(paths, args.output) + diff --git a/misc/tabletool.py b/misc/tabletool.py new file mode 100644 index 0000000..6a18182 --- /dev/null +++ b/misc/tabletool.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +import argparse + +class TableRow: + def __init__(self, cols): + self.values = [None] * cols + + def __len__(self): + return len(self.values) + + def __setitem__(self, index, value): + if index >= len(self.values): + raise IndexError(f"column index {index} out of bounds for row with {len(self.values)} columns") + self.values[index] = value + + def __getitem__(self, index): + if index >= len(self.values): + raise IndexError(f"column index {index} out of bounds for row with {len(self.values)} columns") + return self.values[index] + + def __imul__(self, other): + if len(self.values) != len(other.values): + raise IndexError(f"attempting to perform *= on two table rows with different column counts") + for i in range(len(self)): + self[i] *= other[i] + return self + + def __isub__(self, other): + if len(self.values) != len(other.values): + raise IndexError(f"attempting to perform -= on two table rows with different column counts") + for i in range(len(self)): + self[i] -= other[i] + return self + + def __iadd__(self, other): + if len(self.values) != len(other.values): + raise IndexError(f"attempting to perform += on two table rows with different column counts") + for i in range(len(self)): + self[i] += other[i] + return self + +class Table: + def __init__(self, rows, cols): + self.rows = [TableRow(cols) for i in range(rows)] + + def __getitem__(self, index): + if index >= len(self.rows): + raise IndexError(f"row index {index} out of bounds for table with {len(self.rows)} rows") + return self.rows[index] + + def __imul__(self, other): + if len(self.rows) != len(other.rows): + raise IndexError(f"attempting to perform *= on two tables of different row count") + for selfrow, otherrow in zip(self, other): + selfrow *= otherrow + return self + + def __isub__(self, other): + if len(self.rows) != len(other.rows): + raise IndexError(f"attempting to perform -= on two tables of different row count") + for selfrow, otherrow in zip(self, other): + selfrow -= otherrow + return self + + def __iadd__(self, other): + if len(self.rows) != len(other.rows): + raise IndexError(f"attempting to perform += on two tables of different row count") + for selfrow, otherrow in zip(self, other): + selfrow += otherrow + return self + +def add_table(tables): + lines = [] + EOT = False + EOF = False + while not EOT and not EOF: + line = input().strip() + if line.lower() == "eot": + EOT = True + elif line.lower() == "eof": + EOF = True + else: + lines.append(list(filter(None, line.split()))) + + if len(lines) > 0: + tables.append(lines) + + return EOF + +def parse_tables(tables): + newtables = [] + for table in tables: + t = Table(len(table), max(map(len, table))) + for i, row in enumerate(table): + for j, col in enumerate(table[i]): + t[i][j] = float(table[i][j]) + newtables.append(t) + return newtables + +def perform_operation(tables, op): + first, *tables = tables + for table in tables: + if op == "product": + first *= table + elif op == "difference": + first -= table + elif op == "sum": + first += table + return first + +def print_table(table, precision): + for row in table: + print("\t".join(map(lambda f: str(round(f, precision)), row))) + +if __name__ == "__main__": + argp = argparse.ArgumentParser(description="perform various operations on one or more tables") + + argp.add_argument( + "-o", "--operation", + help="which operation to perform on the inputted tables", + required=True, + choices=["product", "difference", "sum"] + ) + + argp.add_argument( + "-p", "--precision", + help="Number of decimal places the numbers in the outputted table should have", + default=6, + type=int + ) + + args = argp.parse_args() + + print("Input as many tables as you want, but at least two - input 'EOF' (without the quotes) when you are done") + print("For each table, input as many table rows as you want - input 'EOT' (without the quotes) to end a table") + print("For each table row, the columns can be separated by any amount of tabs or spaces") + tables = [] + EOF = False + while not EOF: + EOF = add_table(tables) + tables = parse_tables(tables) + table = perform_operation(tables, args.operation) + print_table(table, args.precision) + diff --git a/scenery/aptdat2airportsxml.py b/scenery/aptdat2airportsxml.py index 09ad3a9..b321190 100644 --- a/scenery/aptdat2airportsxml.py +++ b/scenery/aptdat2airportsxml.py @@ -105,10 +105,10 @@ class Runway: return unit_convert.m2ft(self.get_length_m()) def get_heading1_deg(self): - return self.coord2.angle(self.coord1) + return self.coord1.angle(self.coord2) def get_heading2_deg(self): - return self.coord1.angle(self.coord2) + return self.coord2.angle(self.coord1) def __repr__(self): return f""" @@ -457,7 +457,6 @@ def write_ils_files(ils_d, output, elevpipe, overwrite): i += 1 print() - print(i) if __name__ == "__main__": argp = argparse.ArgumentParser(description="Convert apt.dat files to groundnet.xml files") diff --git a/scenery/coord_convert.py b/scenery/coord_convert.py new file mode 100644 index 0000000..394fb77 --- /dev/null +++ b/scenery/coord_convert.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +import argparse + +def format_skyvector(lon, lat): + lond, lonm = divmod(abs(lon), 1) + lonm = lonm * 60 + latd, latm = divmod(abs(lat), 1) + latm = latm * 60 + ew = "EW"[int(lon < 0)] + ns = "NS"[int(lat < 0)] + return f"{int(latd):02d}{int(latm * 100):04d}{ns}{int(lond):03d}{int(lonm * 100):04d}{ew}" + +if __name__ == "__main__": + argp = argparse.ArgumentParser(description="Convert GPS coordinates between different formats") + + argp.add_argument( + "--lon", + help="Input longitude", + required=True, + type=float + ) + + argp.add_argument( + "--lat", + help="Input latitude", + required=True, + type=float + ) + + argp.add_argument( + "-f", "--format", + help="Output format", + required=True, + choices=["dmd", "dms", "skyvector"] + ) + + args = argp.parse_args() + + if args.format == "skyvector": + result = format_skyvector(args.lon, args.lat) + else: + result = "Output format is not implemented yet" + print(result) +