#!/usr/bin/python3 # # Script to setup a GNS3 project for testing BigBlueButton. # # The project will be centered around a dummy public subnet with both # the bare metal system and most virtual nodes connected to it using # virtual NAT gateways. Servers can be created with the --no-nat # switch to connect them directly to the dummy public subnet. # # "stun.l.google.com" is mimicked with a fake DNS entry so that # BigBlueButton servers see the dummy public subnet as their public IP # address. # # smallstep.com's step-ca server is used to run an ACME Certificate # Authority in a manner compatible with certbot, allowing bbb-install # to run certbot normally. The CA's root certificate and key are # expected to be in the script's directory and are created (using # openssl) if they don't exist. The root certificate is then # installed and trusted on all Ubuntu nodes created by this script. # # All the Ubuntu nodes will be configured to accept the user's public # ssh key and entire authorized_keys file for ssh access. # # BigBlueButton servers with NAT gateways will be configured to forward # port 22 to the server, while the NAT gateway listens for ssh # on port 2222. # # The NAT gateway between the bare metal machine and the dummy public # subnet is given the same name (for ssh access) as the GNS3 project # (default "BigBlueButton"). This node also operates the step-ca and # stun.l.google.com servers. # # The first time the script is run, the required infrastructure nodes # are created. The --public-subnet switch can be specified at this # time, and the public subnet's CIDR prefix will also be saved. # # If the machine running the script is configured to use an APT http # proxy, that proxy will also be used by the created nodes. # # The gns3 library provides "declarative" functions that only create # nodes if they don't already exist, and we use this feature # throughout the script. Running the script on a pre-built network # should change nothing. A single node can be rebuilt by deleting it # and rerunning the script. # RUNTIME DEPENDENCIES # # genisoimage must be installed # openssl must be installed # # USAGE # # ./gns3-bbb.py focal-260 (to install a released server) # ./gns3-bbb.py focal-e41349 (to install a server based on a git commit) # ./gns3-bbb.py testclient (to install a test client) # # By default, the GNS3 project is called "BigBlueButton". If the # project infrastructure (switches and NAT gateways) doesn't exist, it # will be created. # # The '-d' option deletes a single server and its associated subnet and NAT nodes. # # The '--delete-everything' switch deletes EVERYTHING in an existing project. # # See the comments in NPDC/GNS3/gns3.py for more options and how # authentication is done. # # We use an Ubuntu cloud image and a client image created using the # GNS3/ubuntu.py script in BrentBaccala's NPDC github repository. from NPDC.GNS3 import gns3 import sys import os import json import ipaddress import requests import argparse import subprocess SSH_AUTHORIZED_KEYS_FILES = ['~/.ssh/id_rsa.pub', "~/.ssh/authorized_keys"] # These are bootable images provided by Canonical, Inc, that have the cloud-init package # installed. When booted in a VM, cloud-init will configure them based on configuration # provided (in our case) on a ISO image attached to a virtual CD-ROM device. # # Pick up the latest versions from here: # # https://cloud-images.ubuntu.com/releases/bionic/release/ubuntu-18.04-server-cloudimg-amd64.img # https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64.img # https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64.img # # Updated versions are released several times a month. If you don't have the latest version, # this file's cloud-init configuration will run a package update (if package_upgrade is True). cloud_images = { 22: 'ubuntu-22.04-server-cloudimg-amd64.img', 20: 'ubuntu-20.04-server-cloudimg-amd64.img', 18: 'ubuntu-18.04-server-cloudimg-amd64.img' } ubuntu_release = { 20: 'focal', 18: 'bionic' } # We currently use Ubuntu 20 for everything cloud_image = cloud_images[20] # set to True to immediately upgrade to latest package versions; slows thing down a bit package_upgrade = False # Parse the command line options parser = argparse.ArgumentParser(parents=[gns3.parser('BigBlueButton')], description='Start an BigBlueButton test network in GNS3', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--client-image', type=str, help='Ubuntu image to be used for test clients') parser.add_argument('--public-subnet', type=str, help='public IP subnet to be "stolen" for our use (default 128.8.8.0/24)') parser.add_argument('--server-subnet', type=str, default='192.168.1.0/24', help='private IP subnet to be used for NAT-ed BBB server (default 192.168.1.0/24)') parser.add_argument('-r', '--repository', type=str, help='package repository to be used for BigBlueButton server install') parser.add_argument('-g', '--greenlight', action='store_true', help='install Greenlight') parser.add_argument('--ubuntu-release', type=int, default=20, help='Ubuntu release (18 or 20; default 20) to be used for BigBlueButton server install') parser.add_argument('--release', type=str, help='BigBlueButton release to be used for BigBlueButton server install (default is server hostname)') parser.add_argument('--install-script', type=str, help='install script to be used for BigBlueButton server install\n' + 'can be a local file, a URL, or a filename on https://ubuntu.bigbluebutton.org/') parser.add_argument('--proxy-server', type=str, help='proxy server to be passed to BigBlueButton server install script') parser.add_argument('--no-nat', action='store_true', help='install BBB server without a NAT gateway') parser.add_argument('--no-install', action='store_true', help="don't run bbb-install script on BBB server") parser.add_argument('--quiet', default=False, action='store_true', help="don't print console logs to stdout") parser.add_argument('--delete', type=str, help="delete a BBB server and its associated subnet and NAT nodes") parser.add_argument('version', nargs='*', help="""version of BigBlueButton server to be installed (focal-250, focal-25-dev, focal-260, focal-GITREV) version names starting with 'testclient' install clients""") args = parser.parse_args() server_subnet = ipaddress.ip_network(args.server_subnet) if not server_subnet.is_private: print(f"{args.server_subnet} must be a private IP prefix") exit(1) # Various scripts we'll use # # How to open files in the same directory as the script, from https://stackoverflow.com/a/4060259/1493790 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) def file(fn): with open(os.path.join(__location__, fn)) as f: return f.read() # Check if our SSL root Certificate Authority key and certificate are available, create them if not ssl_root_key_fn = os.path.join(__location__, 'bbb-dev-ca.key') ssl_root_crt_fn = os.path.join(__location__, 'bbb-dev-ca.crt') try: with open(ssl_root_key_fn) as f: ssl_root_key = f.read() except: subprocess.run(f'openssl genrsa -out {ssl_root_key_fn} 2048'.split()).check_returncode() with open(ssl_root_key_fn) as f: ssl_root_key = f.read() try: with open(ssl_root_crt_fn) as f: ssl_root_crt = f.read() except: subprocess.run(f'openssl req -x509 -new -nodes -key {ssl_root_key_fn} -sha256 -days 1460 -out {ssl_root_crt_fn} -subj /C=CA/ST=BBB/L=BBB/O=BBB/OU=BBB/CN=BBB-DEV'.split()).check_returncode() with open(ssl_root_crt_fn) as f: ssl_root_crt = f.read() # Open the GNS3 server gns3_server, gns3_project = gns3.open_project_with_standard_options(args) # Extract and validate project's public subnet gns3_variables = gns3_project.variables() if 'public-subnet' in gns3_variables: if not args.public_subnet: args.public_subnet = gns3_variables['public-subnet'] elif args.public_subnet != gns3_variables['public-subnet']: print(f"Public subnet '{args.public_subnet}' doesn't match project's public subnet '{gns3_variables['public-subnet']}'") exit(1) else: if not args.public_subnet: args.public_subnet = '128.8.8.0/24' gns3_variables['public-subnet'] = args.public_subnet gns3_project.set_variables(gns3_variables) public_subnet = ipaddress.ip_network(args.public_subnet) if not public_subnet.is_global: print(f"{args.public_subnet} must be a public IP prefix") exit(1) # Delete a device (and its associated nodes) if that's what we were requested to do if args.delete: # currently, the project's delete method doesn't complain if nothing matches, # so we can just do this, even though some of these devices might not exist # (if the device was created without NAT) gns3_project.delete(args.delete) gns3_project.delete(args.delete + '-NAT') # If the server's subnet was named by its CIDR block, we now have an orphan switch switches = set(node['node_id'] for node in gns3_project.nodes() if node['node_type'] == 'ethernet_switch') linked_switches = set(node['node_id'] for link in gns3_project.links() for node in link['nodes']) for orphan in switches.difference(linked_switches): gns3_project.delete(orphan) exit(0) # Make sure the cloud and client images exist on the GNS3 server if not cloud_image in gns3_server.images(): print(f"{cloud_image} isn't available on GNS3 server {args.host}") exit(1) if not cloud_images[args.ubuntu_release] in gns3_server.images(): print(f"{cloud_images[args.ubuntu_release]} isn't available on GNS3 server {args.host}") exit(1) # An Ubuntu 20 image created by GNS3/ubuntu.py in Brent Baccala's NPDC github repository # # This image comes with the console GUI pre-installed, which cloud_image lacks. # # Default is the most recent ubuntu-open-desktop image (last in the sort order) if any(v.startswith('testclient') for v in args.version): if args.client_image: assert args.client_image in gns3_server.images() else: args.client_image = sorted(image for image in gns3_server.images() if image.startswith('ubuntu-open-desktop'))[-1] # Find out if the system we're running on is configured to use an apt proxy. apt_proxy = None apt_config_command = ['apt-config', '--format', '%f %v%n', 'dump'] apt_config_proc = subprocess.Popen(apt_config_command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) for config_line in apt_config_proc.stdout.read().decode().split('\n'): if ' ' in config_line: key,value = config_line.split(' ', 1) if key == 'Acquire::http::Proxy': apt_proxy = value # Obtain any credentials to authenticate ourself to the VM ssh_authorized_keys = [] for keyfilename in SSH_AUTHORIZED_KEYS_FILES: keyfilename = os.path.expanduser(keyfilename) if os.path.exists(keyfilename): with open(keyfilename) as f: for l in f.read().split('\n'): if l.startswith('ssh-'): ssh_authorized_keys.append(l) ### FUNCTIONS TO CREATE VARIOUS KINDS OF GNS3 OBJECTS def master_gateway(hostname, public_subnet=None, x=0, y=0): # A NAT gateway between our public "Internet" and the actual Internet # # BBB's default STUN server is stun.l.google.com:19302, so we run # coturn and configure the master gateway to mimic it. # # It also operates an ACME server and mimics acme-v02.api.letsencrypt.org, # so our test servers can run certbot to get their SSL certificates. if not isinstance(public_subnet, ipaddress.IPv4Network): raise TypeError("master_gateway() requires public_subnet to be an ipaddress.IPv4Network") # Use the first host address on the subnet for our master gateway # The rest of them will be available for assignment with DHCP public_subnet_hosts = list(public_subnet.hosts()) master_gateway_address = str(public_subnet_hosts[0]) network_config = {'version': 2, 'ethernets': {'ens4': {'dhcp4': 'on', 'dhcp-identifier': 'mac' }, 'ens5': {'addresses': [f'{master_gateway_address}/{public_subnet.prefixlen}'] } }} # resolver1.opendns.com is used by bbb-install to determine external IP address # stun.l.google.com is used by clients to determine their external IP address (and maybe the servers too) # I'd like {acme_server} to be a cname, too, but that isn't good enough (see below) acme_server="acme-v02.api.letsencrypt.org" dnsmasq_conf = f""" listen-address={master_gateway_address} bind-dynamic interface-name=resolver1.opendns.com,ens5 interface-name=stun.l.google.com,ens5 """ # 120 second DHCP lease times because I change things around so # much in the virtual network. I tried 10 second lease times, # but had problems with OSPF route flaps for 10 seconds lease # time on the bare metal system, so it's probably best to # use 120 second leases throughout. dhcpd_conf = f""" ddns-updates on; ddns-update-style standard; update-optimization off; authoritative; # double curlies because it's a Python f-string zone {args.domain}. {{ }} # Update conflict detection prevents a DHCP server from changing # a DNS entry that it didn't create. Since I often delete the # virtual network devices and recreate them, it's best to keep # this turned off. update-conflict-detection off; allow unknown-clients; default-lease-time 120; max-lease-time 120; log-facility local7; subnet {str(public_subnet.network_address)} netmask {str(public_subnet.netmask)} {{ range {str(public_subnet_hosts[1])} {str(public_subnet_hosts[-1])}; option subnet-mask {str(public_subnet.netmask)}; option domain-name-servers {master_gateway_address}; option domain-name "{args.domain}"; option routers {master_gateway_address}; option broadcast-address {str(public_subnet.broadcast_address)}; }} """ # step-ca's JSON configuration file ca = { "root": "/opt/ca/bbb-dev-ca.crt", "crt": "/opt/ca/bbb-dev-ca.crt", "key": "/opt/ca/bbb-dev-ca.key", "address": ":8000", "dnsNames": [ acme_server, hostname, f'{hostname}.{args.domain}', 'localhost' ], "logger": { "format": "text" }, "db": { "type": "badgerv2", "dataSource": "/opt/ca/db" }, "authority": { "provisioners": [ { "type": "ACME", "name": "acme" } ] } } # nginx is used to redirect /directory to /acme/acme/directory, so that certbot can be used # without a server argument. It's configured to mimic {acme_server} nginx_site=f""" server {{ listen 443 ssl http2; listen [::]:443 ssl http2; server_name {acme_server}; ssl_certificate /etc/letsencrypt/live/{acme_server}/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/{acme_server}/privkey.pem; ssl_session_cache shared:SSL:10m; ssl_session_timeout 10m; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; ssl_dhparam /etc/nginx/ssl/dhp-4096.pem; access_log /var/log/nginx/access.log; return 301 https://{acme_server}:8000/acme/acme$request_uri; }} """ # The master gateway's name is the name of the project, and that's # what it announces itself as to DHCP and DNS. user_data = {'hostname': hostname, 'packages': ['dnsmasq', 'isc-dhcp-server', 'coturn', 'bird', 'iptables-persistent', 'nginx', 'python3-certbot-nginx'], 'package_upgrade': package_upgrade, 'users': [{'name': 'ubuntu', 'plain_text_passwd': 'ubuntu', 'ssh_authorized_keys': ssh_authorized_keys, 'lock_passwd': False, 'shell': '/bin/bash', 'sudo': 'ALL=(ALL) NOPASSWD:ALL', }], 'write_files': [ {'path': '/etc/dhcp/dhcpd.conf', 'permissions': '0644', 'content': dhcpd_conf }, {'path': '/etc/dnsmasq.d/gns3-bbb', 'permissions': '0644', 'content': dnsmasq_conf }, {'path': '/etc/bird/bird.conf', 'permissions': '0644', 'content': file('bird.conf') }, {'path': '/etc/systemd/system/step-ca.service', 'permissions': '0644', 'content': file('step-ca.service') }, {'path': '/etc/nginx/sites-available/redirect-ca', 'permissions': '0644', 'content': nginx_site }, {'path': '/opt/ca/ca.json', 'permissions': '0755', 'content': json.dumps(ca) }, {'path': '/opt/ca/bbb-dev-ca.key', 'permissions': '0400', 'content': ssl_root_key }, {'path': '/opt/ca/bbb-dev-ca.crt', 'permissions': '0444', 'content': ssl_root_crt }, {'path': '/usr/local/share/ca-certificates/bbb-dev/bbb-dev-ca.crt', 'permissions': '0444', 'content': ssl_root_crt }, # /var/www/html/bbb-dev-ca.crt should no longer be used for anything {'path': '/var/www/html/bbb-dev-ca.crt', 'permissions': '0444', 'content': ssl_root_crt }, ], 'runcmd': [ # configure coturn to listen on 19302, like stun.l.google.com f'echo aux-server={master_gateway_address}:19302 >> /etc/turnserver.conf', 'systemctl restart coturn', # add CA root certificate from /usr/local/share/ca-certificates 'update-ca-certificates', # now everything we need to operate a certificate authority "wget -q https://dl.step.sm/gh-release/certificates/docs-ca-install/v0.21.0/step-ca_0.21.0_amd64.deb", "dpkg -i step-ca_0.21.0_amd64.deb", "rm step-ca_0.21.0_amd64.deb", # putting {acme_server} in dnsmasq as a cname isn't enough, # because the local step-ca server doesn't do hostname lookups using dnsmasq, # and it will contact {acme_server} to do verification for certbot below f'echo {master_gateway_address} {acme_server} >> /etc/hosts', "systemctl enable step-ca", "systemctl start step-ca", "bash -c 'while ! nc -z localhost 8000; do sleep 1; done'", # get a certificate for nginx f"certbot --server https://localhost:8000/acme/acme/directory certonly --nginx --non-interactive --agree-tos -d {acme_server} -m root@localhost", # complete nginx ssl configuration and restart nginx "mkdir -p /etc/nginx/ssl", "openssl dhparam -dsaparam -out /etc/nginx/ssl/dhp-4096.pem 4096", "ln -s /etc/nginx/sites-available/redirect-ca /etc/nginx/sites-enabled/", "systemctl restart nginx", # enable packet forwarding 'sysctl net.ipv4.ip_forward=1', 'sed -i /net.ipv4.ip_forward=1/s/^#// /etc/sysctl.conf', # enable NAT (both directions) # Connections into the testing network should appear to come from this gateway # on our public "Internet" to allow outside clients to connect to the servers # Connections out of the testing network should appear to come from this gateway # on its bare metal facing virtual subnet for things like package installs 'iptables -t nat -A POSTROUTING -o ens4 -j MASQUERADE', 'iptables -t nat -A POSTROUTING -o ens5 -j MASQUERADE', 'netfilter-persistent save', ], } if notification_url: user_data['phone_home'] = {'url': notification_url, 'tries': 1} # If the system we're running on is configured to use an apt proxy, use it for the NAT instance as well. # # This will break things if the instance can't reach the proxy. if apt_proxy: user_data['apt'] = {'http_proxy': apt_proxy} return gns3_project.ubuntu_node(user_data, image=cloud_image, network_config=network_config, ram=1024, disk=4096, ethernets=2, x=x, y=y) # Initialization server - only used once when project starts to discover project's DNS name # # This server has almost no configuration at all. Once it boots, we can ssh in to find the DNS domain, # which will be used to build the master gateway node, since we need the DNS domain to build its config files. # # Issues: # - if we couldn't find a notification URL, probably will hang. Should at least warn. Improve to fallback on polling # - assumes that this script has access to at least one of the authorized keys def initialization_server(hostname, x=0, y=0): # Use dhcp-identifier: mac because I'm still having problems with # cloned GNS3 ubuntu nodes using the same client identifiers; it's # a cloud-init issue. network_config = {'version': 2, 'ethernets': {'ens4': {'dhcp4': 'on', 'dhcp-identifier': 'mac' }, }} user_data = {'hostname': hostname, 'ssh_authorized_keys': ssh_authorized_keys } if notification_url: user_data['phone_home'] = {'url': notification_url, 'tries': 1} return gns3_project.ubuntu_node(user_data, image=cloud_image, network_config=network_config, ram=512, x=x, y=y) # BigBlueButton TURN server def turn_server(hostname, x=0, y=0): # Use dhcp-identifier: mac because I'm still having problems with # cloned GNS3 ubuntu nodes using the same client identifiers; it's # a cloud-init issue. network_config = {'version': 2, 'ethernets': {'ens4': {'dhcp4': 'on', 'dhcp-identifier': 'mac' }, }} user_data = {'hostname': hostname, 'package_upgrade': package_upgrade, 'users': [{'name': 'ubuntu', 'plain_text_passwd': 'ubuntu', 'ssh_authorized_keys': ssh_authorized_keys, 'lock_passwd': False, 'shell': '/bin/bash', 'sudo': 'ALL=(ALL) NOPASSWD:ALL', }], 'write_files': [ {'path': '/usr/local/share/ca-certificates/bbb-dev/bbb-dev-ca.crt', 'permissions': '0444', 'content': ssl_root_crt }, ], 'runcmd': [ 'update-ca-certificates', ] } if not args.no_install: user_data['runcmd'].append(f'wget -qO- https://ubuntu.bigbluebutton.org/bbb-install.sh | sudo bash -s -- -c {hostname}.{args.domain}:secret -e root@{hostname}.{args.domain}') # If the system we're running on is configured to use an apt proxy, use it for the clients as well. # # This will break things if the instance can't reach the proxy. if apt_proxy: user_data['apt'] = {'http_proxy': apt_proxy} user_data['write_files'].append( {'path': '/etc/apt/apt.conf.d/proxy.conf', 'permissions': '0644', 'content': f'Acquire::http::Proxy "{apt_proxy}";\n' } ) return gns3_project.ubuntu_node(user_data, image=cloud_image, network_config=network_config, ram=1024, disk=4096, x=x, y=y) # BigBlueButton test clients def BBB_client(hostname, x=0, y=0): # Use dhcp-identifier: mac because I'm still having problems with # cloned GNS3 ubuntu nodes using the same client identifiers; it's # a cloud-init issue. network_config = {'version': 2, 'ethernets': {'ens4': {'dhcp4': 'on', 'dhcp-identifier': 'mac', 'optional': True }, 'ens5': {'dhcp4': 'on', 'dhcp-identifier': 'mac', 'optional': True }, 'ens6': {'dhcp4': 'on', 'dhcp-identifier': 'mac', 'optional': True }, }} user_data = {'hostname': hostname, 'package_upgrade': package_upgrade, # We can safely writing files into /home/ubuntu without worrying about its permissions changing # because this is a chained cloud-init that has already booted once and created /home/ubuntu. # If this were an initial boot of a cloud image from Canonical, putting files into /home/ubuntu # would cause that directory's permissions to change to root.root, which would be a problem. 'write_files': [ {'path': '/home/ubuntu/testclient.sh', 'permissions': '0755', 'content': file('testclient.sh') }, {'path': '/usr/local/share/ca-certificates/bbb-dev/bbb-dev-ca.crt', 'permissions': '0444', 'content': ssl_root_crt }, ], 'runcmd': [ 'update-ca-certificates', 'su ubuntu -c /home/ubuntu/testclient.sh', ], } # If the system we're running on is configured to use an apt proxy, use it for the clients as well. # # This will break things if the instance can't reach the proxy. if apt_proxy: user_data['apt'] = {'http_proxy': apt_proxy} user_data['write_files'].append( {'path': '/etc/apt/apt.conf.d/proxy.conf', 'permissions': '0644', 'content': f'Acquire::http::Proxy "{apt_proxy}";\n' } ) # If git user name/email has been set on the current system, it's convenient to set them on the client, too. try: git_user_name = subprocess.check_output('git config --get user.name'.split()).strip().decode() git_user_email = subprocess.check_output('git config --get user.email'.split()).strip().decode() if git_user_name != '': user_data['runcmd'].append(f'su ubuntu -c \'git config --global --add user.name "{git_user_name}"\'') if git_user_email != '': user_data['runcmd'].append(f'su ubuntu -c \'git config --global --add user.email "{git_user_email}"\'') except subprocess.CalledProcessError: pass # need this many virtual CPUs to run the stress tests, which stress the client perhaps more than the server return gns3_project.ubuntu_node(user_data, image=args.client_image, network_config=network_config, cpus=12, ram=8192, disk=8192, ethernets=3, vnc=True, x=x, y=y) # TWO KINDS OF NAT GATEWAYS # A NAT gateway configured for test clients. def client_NAT_gateway(hostname, x=0, y=0, nat_interface='192.168.1.1/24'): interface = ipaddress.ip_interface(nat_interface) hosts = list(interface.network.hosts()) assert hosts[0] == interface.ip # We want DNS service on the NAT gateways so that when we proxy ssh through # them, they can resolve testclient addresses. # This is really broken, but we have to make dnsmasq authoritative # for its DNS domain, or it will hang indefinitely on AAAA # lookups. See the stackexchange link below. We make it # authoritative for the test-client domain (so we can lookup # testclient), but set testclient's domain search path to test, # so it can lookup names like focal-250. # # If we didn't set anything authoritative, AAAA lookups would # hang. If we set the test domain authoritative, server lookups # like focal-250 would return nothing, because they're registered # with the master gateway, not this one. dnsmasq_conf = f""" listen-address={hosts[0]} bind-interfaces dhcp-range={hosts[1]},{hosts[-1]},2m # Don't use dhcp-sequential-ip; assign IP addresses based on hash of client MAC # Otherwise, the IP address change around on reboots, and BBB doesn't like that. # dhcp-sequential-ip dhcp-authoritative # The testclient will register itself with DHCP as 'testclient' # This option will cause dnsmasq to announce it in DNS as 'testclient.test-client' domain=test-client # Make the DNS server authoritative for these domains, or else it will hang # See https://unix.stackexchange.com/questions/720570 auth-zone=test-client auth-zone=in-addr.arpa # auth-server is required when auth-zone is defined; use a non-existent dummy server auth-server=dns.test-client # tell the test clients to search args.domain and not test-client dhcp-option = option:domain-search,{args.domain} """ network_config = {'version': 2, 'ethernets': {'ens4': {'dhcp4': 'on'}, 'ens5': {'addresses': [nat_interface], 'nameservers': {'search' : ['test-client'], 'addresses' : [str(interface.ip)]}}, }} user_data = {'hostname': hostname, 'packages': ['dnsmasq', 'iptables-persistent'], 'package_upgrade': package_upgrade, 'users': [{'name': 'ubuntu', 'plain_text_passwd': 'ubuntu', 'ssh_authorized_keys': ssh_authorized_keys, 'lock_passwd': False, 'shell': '/bin/bash', 'sudo': 'ALL=(ALL) NOPASSWD:ALL', }], 'write_files': [ {'path': '/etc/dnsmasq.d/gns3-bbb', 'permissions': '0644', 'content': dnsmasq_conf }, {'path': '/usr/local/share/ca-certificates/bbb-dev/bbb-dev-ca.crt', 'permissions': '0444', 'content': ssl_root_crt }, ], 'runcmd': [ 'update-ca-certificates', # enable packet forwarding 'sysctl net.ipv4.ip_forward=1', 'sed -i /net.ipv4.ip_forward=1/s/^#// /etc/sysctl.conf', # enable NAT 'iptables -t nat -A POSTROUTING -o ens4 -j MASQUERADE', 'netfilter-persistent save', ], } if notification_url: user_data['phone_home'] = {'url': notification_url, 'tries': 1} # If the system we're running on is configured to use an apt proxy, use it for the NAT instance as well. # # This will break things if the instance can't reach the proxy. if apt_proxy: user_data['apt'] = {'http_proxy': apt_proxy} return gns3_project.ubuntu_node(user_data, image=cloud_image, network_config=network_config, ram=1024, disk=4096, ethernets=2, x=x, y=y) # A NAT gateway for some kind of server (either BigBlueButton or natturn) # between our public "Internet" and a server's private subnet # # 'hostname' should be of the form 'server-NAT', and the NAT gateway will # present itself in DHCP/DNS using the server's name def server_NAT_gateway(hostname, public_subnet=None, x=100, y=100): assert(hostname.endswith('-NAT')) hostname = hostname[:-4] if not isinstance(public_subnet, ipaddress.IPv4Network): raise TypeError("server_NAT_gateway() requires public_subnet to be an ipaddress.IPv4Network") # Use the first host address on the subnet for our master gateway # The rest of them will be available for assignment with DHCP public_subnet_hosts = list(public_subnet.hosts()) master_gateway_address = str(public_subnet_hosts[0]) server_subnet_hosts = list(server_subnet.hosts()) server_nat_address = str(server_subnet_hosts[0]) server_address = str(server_subnet_hosts[1]) first_dhcp_address = str(server_subnet_hosts[2]) last_dhcp_address = str(server_subnet_hosts[-1]) dnsmasq_conf = f""" listen-address={server_nat_address} bind-interfaces dhcp-host={hostname},{server_address} dhcp-range={first_dhcp_address},{last_dhcp_address},2m # Don't use dhcp-sequential-ip; assign IP addresses based on hash of client MAC # Otherwise, the IP address change around on reboots, and BBB doesn't like that. # dhcp-sequential-ip dhcp-authoritative # tell the test servers to search args.domain dhcp-option = option:domain-search,{args.domain} """ # Note that 'hostname-NAT' announces itself into DHCP as 'hostname' # # This is so that the testclients will connect to the NAT gateway to reach the server. network_config = {'version': 2, 'ethernets': {'ens4': {'dhcp4': 'on', 'dhcp4-overrides' : {'hostname' : hostname}}, 'ens5': {'addresses': [f'{server_nat_address}/{server_subnet.prefixlen}']}, }} user_data = {'hostname': hostname + '-NAT', 'packages': ['dnsmasq', 'iptables-persistent'], 'package_upgrade': package_upgrade, 'users': [{'name': 'ubuntu', 'plain_text_passwd': 'ubuntu', 'ssh_authorized_keys': ssh_authorized_keys, 'lock_passwd': False, 'shell': '/bin/bash', 'sudo': 'ALL=(ALL) NOPASSWD:ALL', }], 'write_files': [ {'path': '/etc/dnsmasq.d/gns3-bbb', 'permissions': '0644', 'content': dnsmasq_conf }, {'path': '/usr/local/share/ca-certificates/bbb-dev/bbb-dev-ca.crt', 'permissions': '0444', 'content': ssl_root_crt }, # Our NAT configuration tunnels port 22 through to the BigBlueButton server # Use port 2222 for ssh connections to the server's NAT gateway {'path': '/etc/ssh/sshd_config.d/port.conf', 'permissions': '0644', 'content': "Port 2222" }, ], 'runcmd': [ 'update-ca-certificates', # enable packet forwarding 'sysctl net.ipv4.ip_forward=1', 'sed -i /net.ipv4.ip_forward=1/s/^#// /etc/sysctl.conf', # enable NAT 'iptables -t nat -A POSTROUTING -o ens4 -j MASQUERADE', # These NAT statements assume that the server is on server_address, and relay ssh, web and UDP traffic to it # We ensure that the server is on server_address with a dhcp-host statement in /etc/dnsmasq.conf f'iptables -t nat -A PREROUTING -p tcp -i ens4 --dport 22 -j DNAT --to-destination {server_address}', f'iptables -t nat -A PREROUTING -p tcp -i ens4 --dport 80 -j DNAT --to-destination {server_address}', f'iptables -t nat -A PREROUTING -p tcp -i ens4 --dport 443 -j DNAT --to-destination {server_address}', # port 3478 is TURN; this rule is for NAT gateways fronting TURN servers f'iptables -t nat -A PREROUTING -p tcp -i ens4 --dport 3478 -j DNAT --to-destination {server_address}', f'iptables -t nat -A PREROUTING -p udp -i ens4 --dport 16384:32768 -j DNAT --to-destination {server_address}', # Also, bbb-install requires the server to have the ability to connect to itself using its DNS name, # - we don't know our own IP address at this point # - rule has to be on PREROUTING, else we can't use DNAT # - since it's on PREROUTING, we can't use -o lo # - so grab traffic bound for everything on the public subnet except the master gateway, # which we need to connect to for certbot to work f'iptables -t nat -A PREROUTING -p tcp -d {master_gateway_address} --dport 80 -j ACCEPT', f'iptables -t nat -A PREROUTING -p tcp -d {master_gateway_address} --dport 443 -j ACCEPT', f'iptables -t nat -A PREROUTING -p tcp -d {public_subnet.with_prefixlen} --dport 80 -j DNAT --to-destination {server_address}', f'iptables -t nat -A PREROUTING -p tcp -d {public_subnet.with_prefixlen} --dport 443 -j DNAT --to-destination {server_address}', # hairpin case - we need to rewrite the source address before sending the packets back to the server # no hairpin on port 22; we can ssh into the server, then ssh back to the NAT gateway if needed f'iptables -t nat -A POSTROUTING -s {server_subnet.with_prefixlen} -d {server_address} -p tcp --dport 80 -j MASQUERADE', f'iptables -t nat -A POSTROUTING -s {server_subnet.with_prefixlen} -d {server_address} -p tcp --dport 443 -j MASQUERADE', f'iptables -t nat -A POSTROUTING -s {server_subnet.with_prefixlen} -d {server_address} -p udp --dport 16384:32768 -j MASQUERADE', 'netfilter-persistent save', ], } if notification_url: user_data['phone_home'] = {'url': notification_url, 'tries': 1} # If the system we're running on is configured to use an apt proxy, use it for the NAT instance as well. # # This will break things if the instance can't reach the proxy. if apt_proxy: user_data['apt'] = {'http_proxy': apt_proxy} return gns3_project.ubuntu_node(user_data, image=cloud_image, network_config=network_config, ram=1024, disk=4096, ethernets=2, x=x, y=y) # BIG BLUE BUTTON SERVERS def BBB_server_standalone(hostname, x=100, y=300): # A BigBlueButton server without an associated NAT gateway network_config = {'version': 2, 'ethernets': {'ens4': {'dhcp4': 'on' }}} if not args.release: args.release = hostname if not args.install_script: if '25' in args.release: args.install_script = 'bbb-install-2.5.sh' elif '26' in args.release: args.install_script = 'bbb-install-2.6.sh' else: print("Can't guess which install script version to use") exit(1) if args.install_script.startswith('http:') or args.install_script.startswith('https:'): install_script_request = requests.get(args.install_script) install_script_request.raise_for_status() install_script = install_script_request.text elif '/' in args.install_script: install_script = file(args.install_script) else: install_script_request = requests.get(f"https://ubuntu.bigbluebutton.org/{args.install_script}") install_script_request.raise_for_status() install_script = install_script_request.text user_data = {'hostname': hostname, 'package_upgrade': package_upgrade, 'users': [{'name': 'ubuntu', 'plain_text_passwd': 'ubuntu', 'ssh_authorized_keys': ssh_authorized_keys, 'lock_passwd': False, 'shell': '/bin/bash', 'sudo': 'ALL=(ALL) NOPASSWD:ALL', }], # I'd like to put these files in /home/ubuntu, but then /home/ubuntu will be owned by root # Alternately, I'd put this in /root, but then user ubuntu can't read it 'write_files': [ {'path': '/testserver.sh', 'permissions': '0755', 'content': file('testserver.sh') }, {'path': '/bbb-install.sh', 'permissions': '0755', 'content': install_script }, {'path': '/usr/local/share/ca-certificates/bbb-dev/bbb-dev-ca.crt', 'permissions': '0444', 'content': ssl_root_crt }, ], 'runcmd': [ # add CA root certificate from /usr/local/share/ca-certificates 'update-ca-certificates', ], } if not args.no_install: install_options = [] if args.repository: install_options.append(f'-r {args.repository}') repo = args.repository else: repo = 'ubuntu.bigbluebutton.org' distro = ubuntu_release[args.ubuntu_release] try: requests.head(f'https://{repo}/{args.release}/dists/bigbluebutton-{distro}/Release.gpg').raise_for_status() except requests.exceptions.HTTPError: print(f'Release {args.release} does not exist on {repo} for distribution {distro}') exit(1) if args.proxy_server: install_options.append(f'-p {args.proxy_server}') if args.greenlight: install_options.append('-g') install_options_str = ' '.join(install_options) user_data['runcmd'].append(f'runuser -u ubuntu RELEASE="{args.release}" INSTALL_OPTIONS="{install_options_str}" /testserver.sh') if notification_url: user_data['phone_home'] = {'url': notification_url, 'tries': 1} # If the system we're running on is configured to use an apt proxy, use it for the server as well. # # This will break things if the instance can't reach the proxy. if apt_proxy: user_data['apt'] = {'http_proxy': apt_proxy} user_data['write_files'].append( {'path': '/etc/apt/apt.conf.d/proxy.conf', 'permissions': '0644', 'content': f'Acquire::http::Proxy "{apt_proxy}";\n' } ) return gns3_project.ubuntu_node(user_data, image=cloud_images[args.ubuntu_release], network_config=network_config, cpus=4, ram=8192, disk=16384, x=x, y=y) # BBB server with optional attached NAT gateway def BBB_server(name, x=100, public_subnet=None, depends_on=None): server = BBB_server_standalone(name, x=x, y=300) if args.no_nat: gns3_project.link(server, 0, PublicIP_switch) if depends_on: gns3_project.depends_on(server, depends_on) else: if server_subnet.with_prefixlen not in gns3_project.node_names(): switch = gns3_project.switch(server_subnet.with_prefixlen, x=x, y=200) else: # can't have two gns3 nodes with the same name, so do this instead switch = gns3_project.switch(name + '-subnet', x=x, y=200) server_nat = server_NAT_gateway(name + '-NAT', public_subnet=public_subnet, x=x, y=100) gns3_project.link(server_nat, 0, PublicIP_switch) gns3_project.link(server_nat, 1, switch) gns3_project.link(server, 0, switch) gns3_project.depends_on(server, server_nat) if depends_on: gns3_project.depends_on(server_nat, depends_on) return server # THE VIRTUAL NETWORK # Create a GNS3 "cloud" for Internet access. # # It's done early in the script like this so that the gns3 library # knows which interface we're using, because it might need that # information to construct a notification URL (a global variable). internet = gns3_project.cloud(gns3_variables.get('subnet', args.interface), args.interface, x=-500, y=0) notification_url = gns3_project.notification_url() # Either extract DNS domain from GNS3 variables, or use an init server # to determine the DNS domain and then set it in the project variables # for future reference. if 'domain' in gns3_variables: args.domain = gns3_variables['domain'] if 'domain' not in gns3_variables or 'initsrv' in args.version: init_server = initialization_server('initsrv', x=-200, y=-100) gns3_project.link(init_server, 0, internet) gns3_project.start_nodes(init_server, wait_for_everything=True) # We could ssh to 'ubuntu@initsrv', but that would requiring waiting for the DNS entry to appear, # so let's ssh directly to the IP address ipaddr = gns3_project.httpd.instances_reported['initsrv'] stdout = subprocess.check_output(['ssh', f'ubuntu@{ipaddr}', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', 'netplan ip leases ens4 | grep ^DOMAINNAME= | cut -d = -f 2'], stderr = subprocess.DEVNULL) args.domain = stdout.strip().decode() stdout = subprocess.check_output(['ssh', f'ubuntu@{ipaddr}', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', 'netplan ip leases ens4 | grep ^NETMASK= | cut -d = -f 2'], stderr = subprocess.DEVNULL) veth_netmask = stdout.strip().decode() stdout = subprocess.check_output(['ssh', f'ubuntu@{ipaddr}', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', 'netplan ip leases ens4 | grep ^ADDRESS= | cut -d = -f 2'], stderr = subprocess.DEVNULL) veth_address = stdout.strip().decode() veth_subnet = ipaddress.ip_network(veth_address + "/" + veth_netmask, strict=False) # This isn't atomic; we're using the copy of gns3_variables that we got a minute or so ago gns3_variables['domain'] = args.domain gns3_variables['subnet'] = str(veth_subnet) gns3_project.set_variables(gns3_variables) # If user asked to create just an initsrv, do nothing else if args.version == ['initsrv']: exit(1) # improve delete method so that it can take init_server directly as an arg gns3_project.delete(init_server['node_id']) # Create the master gateway master = master_gateway(args.project, public_subnet=public_subnet, x=-200, y=0) # An Ethernet switch for our public "Internet" PublicIP_switch = gns3_project.switch(public_subnet.with_prefixlen, x=0, y=0, ethernets=16) gns3_project.link(master, 0, internet) gns3_project.link(master, 1, PublicIP_switch) # The BigBlueButton servers and/or test clients for v in args.version: if v == 'turn': # A standard BigBlueButton TURN server turn_node = turn_server('turn', x=-200, y=-200) gns3_project.link(turn_node, 0, PublicIP_switch) gns3_project.depends_on(turn_node, master) elif v == 'natturn': # A BigBlueButton TURN server behind a NAT gateway (like AWS or Azure) if 'natturn' not in gns3_project.node_names(): natturn_nat = server_NAT_gateway('natturn-NAT', public_subnet=public_subnet, x=0, y=-100) if server_subnet.with_prefixlen not in gns3_project.node_names(): natturn_switch = gns3_project.switch(server_subnet.with_prefixlen, x=0, y=-200) else: # can't have two gns3 nodes with the same name, so do this instead natturn_switch = gns3_project.switch('natturn-subnet', x=0, y=-200) natturn_node = turn_server('natturn', x=0, y=-300) gns3_project.link(natturn_nat, 0, PublicIP_switch) gns3_project.link(natturn_nat, 1, natturn_switch) gns3_project.link(natturn_node, 0, natturn_switch) gns3_project.depends_on(natturn_node, natturn_nat) gns3_project.depends_on(natturn_nat, master) elif v.startswith('testclient'): # Create client NAT gateways first # Remember, they won't be created if they already exist # # NAT4: public subnet to carrier grade NAT subnet # # We put a switch on here to ensure that NAT6's interface will be up when it boots. # Otherwise, if the interface is down, it won't start its DHCP server (ever). # # NAT4, NAT5, and NAT6 are numbered to match the corresponding 'ens[456]' # interface names on 'testclient'. subnet = '100.64.1.1/24' nat4 = client_NAT_gateway('NAT4', x=150, y=-200, nat_interface=subnet) gns3_project.link(nat4, 0, PublicIP_switch) nat4_switch = gns3_project.switch(subnet, x=350, y=-200) gns3_project.link(nat4, 1, nat4_switch) gns3_project.depends_on(nat4, master) # NAT5: public subnet to private client subnet, not overlapping server address space # # Put a switch on here for the same reason as NAT4. subnet = '192.168.128.1/24' nat5 = client_NAT_gateway('NAT5', x=150, y=-100, nat_interface=subnet) gns3_project.link(nat5, 0, PublicIP_switch) nat5_switch = gns3_project.switch(subnet, x=350, y=-100) gns3_project.link(nat5, 1, nat5_switch) gns3_project.depends_on(nat5, master) # NAT6: public subnet to private client subnet, overlapping server address space # # Put a switch on here for the same reason as NAT4. subnet = '192.168.1.1/24' nat6 = client_NAT_gateway('NAT6', x=150, y=0, nat_interface=subnet) gns3_project.link(nat6, 0, PublicIP_switch) nat6_switch = gns3_project.switch(subnet, x=350, y=0) gns3_project.link(nat6, 1, nat6_switch) gns3_project.depends_on(nat6, master) # Create actual test client, now # find an unoccupied y coordinate on the GUI for y in (-200, -100, 0, 100, 200): try: next(n for n in gns3_project.nodes() if n['x'] == 550 and n['y'] == y) except StopIteration: break client = BBB_client(v, x=550, y=y) gns3_project.link(client, 0, nat4_switch) gns3_project.depends_on(client, nat4) gns3_project.link(client, 1, nat5_switch) gns3_project.depends_on(client, nat5) gns3_project.link(client, 2, nat6_switch) gns3_project.depends_on(client, nat6) else: # Create a server # find an spot on the GUI if args.no_nat: y_coordinates_needed = [300] else: y_coordinates_needed = [100, 200, 300] for x in (0, 200, -200, 400, -400): try: next(n for n in gns3_project.nodes() if n['x'] == x and n['y'] in y_coordinates_needed) except StopIteration: break BBB_server(v, x=x, public_subnet=public_subnet, depends_on=master) # The difference between these two is that start_nodes waits for notification that # the nodes booted, while start_node does not. # # The project might not have a notification_url if the script couldn't figure out # a local IP address suitable for a callback. if notification_url: gns3_project.start_nodes(quiet=args.quiet) else: for node in args.version: gns3_project.start_node(node, quiet=args.quiet)