FGFS-4.1/phpvms
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace importer with AJAX powered; better error handling #443 (#447)

Browse Source 
* Replace importer with AJAX powered; better error handling #443

* Formatting

* Fix command line importer
This commit is contained in:
Nabeel S 2019-12-02 09:57:35 -05:00 committed by GitHub
parent 50dc79bc8d
commit 68eff40753
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 544 additions and 354 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/Contracts/Model.php
View File

@ -6,8 +6,10 @@ namespace App\Contracts;
* @property mixed $id
* @property bool $skip_mutator
*
* @method static Model find(int $airline_id)
* @method static create(array $attrs)
* @method static Model find(int $id)
* @method static Model where(array $array)
* @method static Model firstOrCreate(array $where, array $array)
* @method static Model updateOrCreate(array $array, array $attrs)
* @method static truncate()
*/

1
app/Models/Aircraft.php
View File

@ -6,6 +6,7 @@ use App\Contracts\Model;
use App\Models\Enums\AircraftStatus;
use App\Models\Traits\ExpensableTrait;
use App\Models\Traits\FilesTrait;
use Carbon\Carbon;
/**
* @property int id

11
app/Providers/AppServiceProvider.php
View File

@ -25,6 +25,7 @@ use App\Models\Subfleet;
use App\Models\User;
use App\Repositories\SettingRepository;
use App\Services\ModuleService;
use Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\View;
use Illuminate\Support\ServiceProvider;
@ -66,13 +67,11 @@ class AppServiceProvider extends ServiceProvider
*/
public function register(): void
{
// Only dev environment stuff
if ($this->app->environment() === 'dev') {
// Only load the IDE helper if it's included. This lets use distribute the
// package without any dev dependencies
// Only load the IDE helper if it's included and enabled
if (config('app.debug_toolbar') === true) {
/* @noinspection NestedPositiveIfStatementsInspection */
if (class_exists(\Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider::class)) {
$this->app->register(\Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider::class);
if (class_exists(IdeHelperServiceProvider::class)) {
$this->app->register(IdeHelperServiceProvider::class);
}
}
}

1
config/app.php
View File

@ -13,6 +13,7 @@ return [
'debug' => env('APP_DEBUG', true),
'url' => env('APP_URL', 'http://localhost'),
'version' => '7.0.0',
'debug_toolbar' => false,
'locale' => env('APP_LOCALE', 'en'),
'fallback_locale' => 'en',

17
modules/Installer/Config/config.php
View File

@ -1,12 +1,5 @@
<?php
use Modules\Installer\Services\Importer\Stages\Stage1;
use Modules\Installer\Services\Importer\Stages\Stage2;
use Modules\Installer\Services\Importer\Stages\Stage3;
use Modules\Installer\Services\Importer\Stages\Stage4;
use Modules\Installer\Services\Importer\Stages\Stage5;
use Modules\Installer\Services\Importer\Stages\Stage6;
return [
'php' => [
'version' => '7.2',
@ -46,14 +39,6 @@ return [
],
'importer' => [
'batch_size' => 150,
'stages' => [
'stage1' => Stage1::class,
'stage2' => Stage2::class,
'stage3' => Stage3::class,
'stage4' => Stage4::class,
'stage5' => Stage5::class,
'stage6' => Stage6::class,
],
'batch_size' => 20,
],
];

23
modules/Installer/Console/Commands/ImportFromClassicCommand.php
View File

@ -4,8 +4,6 @@ namespace Modules\Installer\Console\Commands;
use App\Contracts\Command;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Services\Importer\ImporterService;
class ImportFromClassicCommand extends Command
@ -29,24 +27,13 @@ class ImportFromClassicCommand extends Command
$importerSvc = new ImporterService();
$importerSvc->saveCredentials($creds);
$manifest = $importerSvc->generateImportManifest();
$stage = 'stage1';
$start = 0;
while (true) {
foreach ($manifest as $record) {
try {
$importerSvc->run($stage, $start);
} catch (ImporterNextRecordSet $e) {
Log::info('More records, starting from '.$e->nextOffset);
$start = $e->nextOffset;
} catch (StageCompleted $e) {
$stage = $e->nextStage;
$start = 0;
Log::info('Stage '.$stage.' completed, moving to '.$e->nextStage);
if ($e->nextStage === 'complete') {
break;
}
$importerSvc->run($record['importer'], $record['start']);
} catch (\Exception $e) {
Log::error($e->getMessage());
}
}
}

59
modules/Installer/Http/Controllers/ImporterController.php
View File

@ -5,8 +5,6 @@ namespace Modules\Installer\Http\Controllers;
use App\Contracts\Controller;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Services\Importer\ImporterService;
class ImporterController extends Controller
@ -28,6 +26,8 @@ class ImporterController extends Controller
*/
public function index(Request $request)
{
app('debugbar')->disable(); // saves the query logging
return view('installer::importer/step1-configure');
}
@ -40,11 +40,26 @@ class ImporterController extends Controller
*/
public function config(Request $request)
{
app('debugbar')->disable(); // saves the query logging
try {
// Save the credentials to use later
$this->importerSvc->saveCredentialsFromRequest($request);
// Generate the import manifest
$manifest = $this->importerSvc->generateImportManifest();
} catch (\Exception $e) {
Log::error($e->getMessage());
// Send it to run, step1
return redirect(route('importer.run').'?stage=stage1&start=0');
return view('installer::importer/error', [
'error' => $e->getMessage(),
]);
}
// Send it to run, step1
return view('installer::importer/step2-processing', [
'manifest' => $manifest,
]);
}
/**
@ -55,33 +70,31 @@ class ImporterController extends Controller
*
* @param \Illuminate\Http\Request $request
*
* @throws \Exception
*
* @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
*/
public function run(Request $request)
{
$stage = $request->get('stage');
$start = $request->get('start');
app('debugbar')->disable(); // saves the query logging
Log::info('Starting stage '.$stage.' from offset '.$start);
$importer = $request->input('importer');
$start = $request->input('start');
try {
$this->importerSvc->run($stage, $start);
Log::info('Starting stage '.$importer.' from offset '.$start);
$this->importerSvc->run($importer, $start);
return response()->json([
'message' => 'completed',
]);
}
// The importer wants to move onto the next set of records, so refresh this page and continue
catch (ImporterNextRecordSet $e) {
Log::info('Getting more records for stage '.$stage.', starting at '.$e->nextOffset);
return redirect(route('importer.run').'?stage='.$stage.'&start='.$e->nextOffset);
}
// This stage is completed, so move onto the next one
catch (StageCompleted $e) {
if ($e->nextStage === 'complete') {
return view('installer::importer/complete');
}
Log::info('Completed stage '.$stage.', redirect to '.$e->nextStage);
return redirect(route('importer.run').'?stage='.$e->nextStage.'&start=0');
}
/**
* Complete the import
*/
public function complete()
{
return redirect('/');
}
}

4
modules/Installer/Http/Routes/importer.php
View File

@ -4,6 +4,6 @@ use Illuminate\Support\Facades\Route;
Route::get('/', 'ImporterController@index')->name('index');
Route::post('/config', 'ImporterController@config')->name('config');
Route::get('/run', 'ImporterController@run')->name('run');
Route::post('/run', 'ImporterController@run')->middleware('api')->name('run');
Route::get('/complete', 'ImporterController@complete')->name('complete');
Route::post('/complete', 'ImporterController@complete')->name('complete');

6
modules/Installer/Resources/views/app.blade.php
View File

@ -12,6 +12,9 @@
<meta content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0, shrink-to-fit=no'
name='viewport'/>
<meta name="base-url" content="{!! url('') !!}">
<meta name="api-key" content="{!! Auth::check() ? Auth::user()->api_key: '' !!}">
<meta name="csrf-token" content="{!! csrf_token() !!}">
<link href="https://fonts.googleapis.com/css?family=Montserrat:400,700,200" rel="stylesheet"/>
<link href="https://maxcdn.bootstrapcdn.com/font-awesome/latest/css/font-awesome.min.css" rel="stylesheet"/>
@ -63,6 +66,9 @@
{{--<script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js"></script>--}}
<script src="{{ public_mix('/assets/global/js/vendor.js') }}"></script>
<script src="{{ public_mix('/assets/frontend/js/vendor.js') }}"></script>
<script src="{{ public_mix('/assets/frontend/js/app.js') }}"></script>
<script src="{{ public_asset('/assets/installer/js/vendor.js') }}" type="text/javascript"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>

9
modules/Installer/Resources/views/importer/error.blade.php Normal file
View File

@ -0,0 +1,9 @@
@extends('installer::app')
@section('title', 'Import Error!')
@section('content')
<div style="align-content: center;">
<h4>Error!</h4>
<p class="text-danger">{{ $error }}</p>
</div>
@endsection

2
modules/Installer/Resources/views/importer/step1-configure.blade.php
View File

@ -117,7 +117,7 @@
e.preventDefault();
const opts = {
_token: "{{ csrf_token() }}",
db_conn: $("#db_conn option:selected").text(),
db_conn: 'mysql',
db_host: $("input[name=db_host]").val(),
db_port: $("input[name=db_port]").val(),
db_name: $("input[name=db_name]").val(),

171
modules/Installer/Resources/views/importer/step2-processing.blade.php Normal file
View File

@ -0,0 +1,171 @@
@extends('installer::app')
@section('title', 'Import Configuration')
@section('content')
<div style="align-content: center;">
{{ Form::open(['route' => 'importer.complete', 'method' => 'POST']) }}
<table class="table" width="25%">
<tr>
<td colspan="2"><h4>Running Importer</h4></td>
</tr>
<tr>
<td colspan="2">
<div class="progress">
<div id="progress" class="progress-bar" style="width: 0%"></div>
</div>
<div>
<p id="message" style="margin-top: 7px;"></p>
<p id="error" class="text-danger" style="margin-top: 7px;"></p>
</div>
</td>
</tr>
</table>
<p style="text-align: right">
{{ Form::submit('Complete Import', [
'id' => 'completebutton',
'class' => 'btn btn-success'
]) }}
</p>
{{ Form::close() }}
</div>
@endsection
@section('scripts')
<script>
const manifest = {!!json_encode($manifest) !!};
/**
* Get the total number of steps
*/
const getTotalSteps = () => {
let total = 0;
for (let importer of manifest_keys) {
for (let stage of manifest[importer]) {
total++;
}
}
return total;
};
/**
* Run each step of the importer
*/
async function startImporter() {
let current = 1;
const total_steps = manifest.length;
/**
* Update the progress bar
*/
const setProgress = (current, message) => {
const percent = Math.round(current / total_steps * 100);
$("#progress").css("width", `${percent}%`);
$("#message").text(message);
};
/**
* Sleep for a given interval
*/
const sleep = (timeout) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve();
}, timeout);
});
};
const setError = (error) => {
let message = '';
if (error.response) {
// The request was made and the server responded with a status code
// that falls out of the range of 2xx
console.log(error.response.data);
console.log(error.response.status);
console.log(error.response.headers);
message = error.response.data.message;
} else if (error.request) {
// The request was made but no response was received
// `error.request` is an instance of XMLHttpRequest in the browser and an instance of
// http.ClientRequest in node.js
console.log(error.request);
message = error.request;
} else {
// Something happened in setting up the request that triggered an Error
console.log('Error', error.message);
message = error.message;
}
$("#error").text(`Error processing, check the logs: ${message}`);
console.log(error.config);
};
/**
* Call the endpoint as a POST
*/
const runStep = async function (stage) {
setProgress(current, stage.message);
try {
return await phpvms.request({
method: 'post',
url: '/importer/run',
data: {
importer: stage.importer,
start: stage.start,
}
});
} catch (e) {
if (e.response.status === 504) {
const err = $("#error");
console.log('got timeout, retrying');
err.text(`Timed out, attempting to retry`);
// await sleep(5000);
const val = await runStep(stage);
err.text('');
return val;
}
setError(e);
throw e;
}
};
let errors = false;
const complete_button = $("#completebutton");
complete_button.hide();
for (let stage of manifest) {
console.log(`Running ${stage.importer} step ${stage.start}`);
try {
await runStep(stage);
} catch (e) {
errors = true;
break;
}
current++;
}
if (!errors) {
$("#message").text('Done!');
complete_button.show();
}
}
$(document).ready(() => {
startImporter();
});
</script>
@endsection

35
modules/Installer/Services/DatabaseService.php
View File

@ -3,7 +3,8 @@
namespace Modules\Installer\Services;
use App\Contracts\Service;
use Log;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\Log;
use PDO;
class DatabaseService extends Service
@ -24,7 +25,20 @@ class DatabaseService extends Service
{
Log::info('Testing Connection: '.$driver.'::'.$user.':<hidden>@'.$host.':'.$port.';'.$name);
if ($driver === 'mysql') {
// TODO: Needs testing
if ($driver === 'postgres') {
$dsn = "pgsql:host=$host;port=$port;dbname=$name";
try {
$conn = new PDO($dsn, $user, $pass);
} catch (\PDOException $e) {
throw $e;
}
return true;
}
// Default MySQL
$dsn = "mysql:host=$host;port=$port;dbname=$name";
Log::info('Connection string: '.$dsn);
@ -33,19 +47,6 @@ class DatabaseService extends Service
} catch (\PDOException $e) {
throw $e;
}
}
// TODO: Needs testing
elseif ($driver === 'postgres') {
$dsn = "pgsql:host=$host;port=$port;dbname=$name";
try {
$conn = new PDO($dsn, $user, $pass);
} catch (\PDOException $e) {
throw $e;
}
}
return true;
}
@ -59,8 +60,8 @@ class DatabaseService extends Service
$output = '';
if (config('database.default') === 'sqlite') {
\Artisan::call('database:create');
$output .= \Artisan::output();
Artisan::call('database:create');
$output .= Artisan::output();
}
return trim($output);

48
modules/Installer/Services/Importer/BaseImporter.php
View File

@ -7,8 +7,6 @@ use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Modules\Installer\Utils\IdMapper;
use Modules\Installer\Utils\ImporterDB;
use Modules\Installer\Utils\LoggerTrait;
@ -19,6 +17,7 @@ abstract class BaseImporter implements ShouldQueue
protected $db;
protected $idMapper;
protected $table;
public function __construct()
{
@ -32,13 +31,52 @@ abstract class BaseImporter implements ShouldQueue
*
* @param int $start
*
* @throws ImporterNoMoreRecords
* @throws ImporterNextRecordSet
*
* @return mixed
*/
abstract public function run($start = 0);
/**
* Return a manifest of the import tasks to run. Returns an array of objects,
* which contain a start and end row
*
* @return array
*/
public function getManifest(): array
{
$manifest = [];
$start = 0;
$total_rows = $this->db->getTotalRows($this->table);
do {
$end = $start + $this->db->batchSize;
if ($end > $total_rows) {
$end = $total_rows;
}
$idx = $start + 1;
$manifest[] = [
'importer' => get_class($this),
'start' => $start,
'end' => $end,
'message' => 'Importing '.$this->table.' ('.$idx.' - '.$end.' of '.$total_rows.')',
];
$start += $this->db->batchSize;
} while ($start < $total_rows);
return $manifest;
}
/**
* Determine what columns exist, can be used for feature testing between v2/v5
*
* @return array
*/
public function getColumns(): array
{
}
/**
* @param $date
*

68
modules/Installer/Services/Importer/BaseStage.php
View File

@ -1,68 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer;
use App\Repositories\KvpRepository;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Utils\IdMapper;
use Modules\Installer\Utils\ImporterDB;
use Modules\Installer\Utils\LoggerTrait;
class BaseStage
{
use LoggerTrait;
public $importers = [];
public $nextStage = '';
protected $db;
protected $idMapper;
/**
* @var KvpRepository
*/
protected $kvpRepo;
public function __construct(ImporterDB $db, IdMapper $mapper)
{
$this->db = $db;
$this->idMapper = $mapper;
$this->kvpRepo = app(KvpRepository::class);
}
/**
* Run all of the given importers
*
* @param $start
*
* @throws ImporterNextRecordSet
* @throws StageCompleted
*/
public function run($start)
{
$importersRun = $this->kvpRepo->get('importers.run', []);
foreach ($this->importers as $klass) {
/** @var $importer \Modules\Installer\Services\Importer\BaseImporter */
$importer = new $klass($this->db, $this->idMapper);
try {
$importer->run($start);
} catch (ImporterNextRecordSet $e) {
Log::info('Requesting next set of records');
throw $e;
} catch (ImporterNoMoreRecords $e) {
$importersRun[] = $importer;
}
}
$this->kvpRepo->save('importers.run', $importersRun);
throw new StageCompleted($this->nextStage);
}
}

76
modules/Installer/Services/Importer/ImporterService.php
View File

@ -4,11 +4,18 @@ namespace Modules\Installer\Services\Importer;
use App\Contracts\Service;
use App\Repositories\KvpRepository;
use Exception;
use Illuminate\Http\Request;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Utils\IdMapper;
use Modules\Installer\Utils\ImporterDB;
use Modules\Installer\Services\Importer\Importers\AircraftImporter;
use Modules\Installer\Services\Importer\Importers\AirlineImporter;
use Modules\Installer\Services\Importer\Importers\AirportImporter;
use Modules\Installer\Services\Importer\Importers\ClearDatabase;
use Modules\Installer\Services\Importer\Importers\FinalizeImporter;
use Modules\Installer\Services\Importer\Importers\FlightImporter;
use Modules\Installer\Services\Importer\Importers\GroupImporter;
use Modules\Installer\Services\Importer\Importers\PirepImporter;
use Modules\Installer\Services\Importer\Importers\RankImport;
use Modules\Installer\Services\Importer\Importers\UserImport;
class ImporterService extends Service
{
@ -20,22 +27,23 @@ class ImporterService extends Service
private $kvpRepo;
/**
* Hold some of our data on disk for the migration
*
* @var IdMapper
* The list of importers, in proper order
*/
private $idMapper;
/**
* Hold the PDO connection to the old database
*
* @var ImporterDB
*/
private $db;
private $importList = [
ClearDatabase::class,
RankImport::class,
GroupImporter::class,
AirlineImporter::class,
AircraftImporter::class,
AirportImporter::class,
FlightImporter::class,
UserImport::class,
PirepImporter::class,
FinalizeImporter::class,
];
public function __construct()
{
$this->idMapper = app(IdMapper::class);
$this->kvpRepo = app(KvpRepository::class);
}
@ -87,25 +95,41 @@ class ImporterService extends Service
return $this->kvpRepo->get($this->CREDENTIALS_KEY);
}
/**
* Create a manifest of the import. Creates an array with the importer name,
* which then has a subarray of all of the different steps/stages it needs to run
*/
public function generateImportManifest()
{
$manifest = [];
foreach ($this->importList as $importerKlass) {
/** @var \Modules\Installer\Services\Importer\BaseImporter $importer */
$importer = new $importerKlass();
$manifest = array_merge($manifest, $importer->getManifest());
}
return $manifest;
}
/**
* Run a given stage
*
* @param $stage
* @param $importer
* @param int $start
*
* @throws ImporterNextRecordSet
* @throws StageCompleted
* @throws \Exception
*
* @return int|void
*/
public function run($stage, $start = 0)
public function run($importer, $start = 0)
{
$db = new ImporterDB($this->kvpRepo->get($this->CREDENTIALS_KEY));
if (!in_array($importer, $this->importList)) {
throw new Exception('Unknown importer "'.$importer.'"');
}
$stageKlass = config('installer.importer.stages.'.$stage);
/** @var $stage \Modules\Installer\Services\Importer\BaseStage */
$stage = new $stageKlass($db, $this->idMapper);
$stage->run($start);
/** @var $importerInst \Modules\Installer\Services\Importer\BaseImporter */
$importerInst = new $importer();
$importerInst->run($start);
}
}

12
modules/Installer/Services/Importer/Importers/AircraftImporter.php
View File

@ -5,21 +5,17 @@ namespace Modules\Installer\Services\Importer\Importers;
use App\Models\Aircraft;
use App\Models\Airline;
use App\Models\Subfleet;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Modules\Installer\Services\Importer\BaseImporter;
class AircraftImporter extends BaseImporter
{
public $table = 'aircraft';
/**
* CONSTANTS
*/
public const SUBFLEET_NAME = 'Imported Aircraft';
/**
* @param int $start
*
* @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
*/
public function run($start = 0)
{
$this->comment('--- AIRCRAFT IMPORT ---');
@ -29,7 +25,7 @@ class AircraftImporter extends BaseImporter
$this->info('Subfleet ID is '.$subfleet->id);
$count = 0;
foreach ($this->db->readRows('aircraft') as $row) {
foreach ($this->db->readRows($this->table, $start) as $row) {
$where = [
'name' => $row->fullname,
'registration' => $row->registration,
@ -49,8 +45,6 @@ class AircraftImporter extends BaseImporter
}
$this->info('Imported '.$count.' aircraft');
throw new ImporterNoMoreRecords();
}
/**

18
modules/Installer/Services/Importer/Importers/AirlineImporter.php
View File

@ -4,27 +4,31 @@ namespace Modules\Installer\Services\Importer\Importers;
use App\Models\Airline;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Modules\Installer\Services\Importer\BaseImporter;
class AirlineImporter extends BaseImporter
{
public $table = 'airlines';
/**
* @param int $start
*
* @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
*/
public function run($start = 0)
{
$this->comment('--- AIRLINE IMPORT ---');
$count = 0;
foreach ($this->db->readRows('airlines', $start) as $row) {
$airline = Airline::firstOrCreate(['icao' => $row->code], [
foreach ($this->db->readRows($this->table, $start) as $row) {
$attrs = [
'iata' => $row->code,
'name' => $row->name,
'active' => $row->enabled,
]);
];
$w = ['icao' => $row->code];
//$airline = Airline::firstOrCreate($w, $attrs);
$airline = Airline::create(array_merge($w, $attrs));
$this->idMapper->addMapping('airlines', $row->id, $airline->id);
$this->idMapper->addMapping('airlines', $row->code, $airline->id);
@ -37,7 +41,5 @@ class AirlineImporter extends BaseImporter
}
$this->info('Imported '.$count.' airlines');
throw new ImporterNoMoreRecords();
}
}

39
modules/Installer/Services/Importer/Importers/AirportImporter.php
View File

@ -3,22 +3,29 @@
namespace Modules\Installer\Services\Importer\Importers;
use App\Models\Airport;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Illuminate\Database\QueryException;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Services\Importer\BaseImporter;
class AirportImporter extends BaseImporter
{
/**
* @param int $start
*
* @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
*/
protected $table = 'airports';
public function run($start = 0)
{
$this->comment('--- AIRPORT IMPORT ---');
$fields = [
'icao',
'name',
'country',
'lat',
'lng',
'hub',
];
$count = 0;
foreach ($this->db->readRows('airports', $start) as $row) {
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$attrs = [
'id' => trim($row->icao),
'icao' => trim($row->icao),
@ -29,7 +36,21 @@ class AirportImporter extends BaseImporter
'hub' => $row->hub,
];
$airport = Airport::updateOrCreate(['id' => $attrs['id']], $attrs);
$w = ['id' => $attrs['id']];
//$airport = Airport::updateOrCreate($w, $attrs);
try {
$airport = Airport::create(array_merge($w, $attrs));
} catch (QueryException $e) {
$sqlState = $e->errorInfo[0];
$errorCode = $e->errorInfo[1];
if ($sqlState === '23000' && $errorCode === 1062) {
Log::info('Found duplicate for '.$row->icao.', ignoring');
return true;
}
return false;
}
if ($airport->wasRecentlyCreated) {
$count++;
@ -37,7 +58,5 @@ class AirportImporter extends BaseImporter
}
$this->info('Imported '.$count.' airports');
throw new ImporterNoMoreRecords();
}
}

47
modules/Installer/Services/Importer/Stages/Stage1.php → modules/Installer/Services/Importer/Importers/ClearDatabase.php
View File

@ -1,6 +1,6 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
namespace Modules\Installer\Services\Importer\Importers;
use App\Models\Acars;
use App\Models\Airline;
@ -19,37 +19,28 @@ use App\Models\Subfleet;
use App\Models\User;
use App\Models\UserAward;
use Illuminate\Support\Facades\DB;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\AircraftImporter;
use Modules\Installer\Services\Importer\Importers\AirlineImporter;
use Modules\Installer\Services\Importer\Importers\GroupImporter;
use Modules\Installer\Services\Importer\Importers\RankImport;
use Modules\Installer\Services\Importer\BaseImporter;
class Stage1 extends BaseStage
class ClearDatabase extends BaseImporter
{
public $importers = [
RankImport::class,
AirlineImporter::class,
AircraftImporter::class,
GroupImporter::class,
];
public $nextStage = 'stage2';
/**
* @param int $start Record number to start from
*
* @throws ImporterNextRecordSet
* @throws StageCompleted
* Returns a default manifest just so this step gets run
*/
public function getManifest(): array
{
return [
[
'importer' => get_class($this),
'start' => 0,
'end' => 1,
'message' => 'Clearing database',
],
];
}
public function run($start = 0)
{
$this->cleanupDb();
// Run the first set of importers
parent::run($start);
}
/**
@ -60,6 +51,8 @@ class Stage1 extends BaseStage
{
$this->info('Running database cleanup/empty before starting');
DB::statement('SET FOREIGN_KEY_CHECKS=0');
Bid::truncate();
File::truncate();
News::truncate();
@ -90,8 +83,6 @@ class Stage1 extends BaseStage
UserAward::truncate();
User::truncate();
// Re-run the base seeds
//$seederSvc = app(SeederService::class);
//$seederSvc->syncAllSeeds();
DB::statement('SET FOREIGN_KEY_CHECKS=1');
}
}

31
modules/Installer/Services/Importer/Stages/Stage6.php → modules/Installer/Services/Importer/Importers/FinalizeImporter.php
View File

@ -1,22 +1,39 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
namespace Modules\Installer\Services\Importer\Importers;
use App\Models\User;
use App\Services\UserService;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\BaseImporter;
class Stage6 extends BaseStage
class FinalizeImporter extends BaseImporter
{
public $nextStage = 'complete';
/**
* Returns a default manifest just so this step gets run
*/
public function getManifest(): array
{
return [
[
'importer' => get_class($this),
'start' => 0,
'end' => 1,
'message' => 'Finalizing import',
],
];
}
/**
* The start method. Takes the offset to start from
*
* @param int $start
*
* @return mixed
*/
public function run($start = 0)
{
$this->findLastPireps();
$this->recalculateUserStats();
throw new StageCompleted($this->nextStage);
}
/**

23
modules/Installer/Services/Importer/Importers/FlightImporter.php
View File

@ -7,12 +7,30 @@ use Modules\Installer\Services\Importer\BaseImporter;
class FlightImporter extends BaseImporter
{
protected $table = 'schedules';
public function run($start = 0)
{
$this->comment('--- FLIGHT SCHEDULE IMPORT ---');
$fields = [
'id',
'code',
'flightnum',
'depicao',
'arricao',
'route',
'distance',
'flightlevel',
'deptime',
'arrtime',
'flightttime',
'notes',
'enabled',
];
$count = 0;
foreach ($this->db->readRows('schedules', $start) as $row) {
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$airline_id = $this->idMapper->getMapping('airlines', $row->code);
$flight_num = trim($row->flightnum);
@ -32,7 +50,8 @@ class FlightImporter extends BaseImporter
try {
$w = ['airline_id' => $airline_id, 'flight_number' => $flight_num];
$flight = Flight::updateOrCreate($w, $attrs);
// $flight = Flight::updateOrCreate($w, $attrs);
$flight = Flight::create(array_merge($w, $attrs));
} catch (\Exception $e) {
//$this->error($e);
}

4
modules/Installer/Services/Importer/Importers/GroupImporter.php
View File

@ -11,6 +11,8 @@ use Modules\Installer\Services\Importer\BaseImporter;
*/
class GroupImporter extends BaseImporter
{
protected $table = 'groups';
/**
* Permissions in the legacy system, mapping them to the current system
*/
@ -66,7 +68,7 @@ class GroupImporter extends BaseImporter
$roleSvc = app(RoleService::class);
$count = 0;
foreach ($this->db->readRows('groups') as $row) {
foreach ($this->db->readRows($this->table, $start) as $row) {
$name = str_slug($row->name);
$role = Role::firstOrCreate(
['name' => $name],

28
modules/Installer/Services/Importer/Importers/PirepImporter.php
View File

@ -10,12 +10,33 @@ use Modules\Installer\Services\Importer\BaseImporter;
class PirepImporter extends BaseImporter
{
protected $table = 'pireps';
public function run($start = 0)
{
$this->comment('--- PIREP IMPORT ---');
$fields = [
'pirepid',
'pilotid',
'code',
'aircraft',
'flightnum',
'depicao',
'arricao',
'fuelused',
'route',
'source',
'accepted',
'submitdate',
'distance',
'flighttime_stamp',
'flighttype',
'flightlevel',
];
$count = 0;
foreach ($this->db->readRows('pireps', $start) as $row) {
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$pirep_id = $row->pirepid;
$user_id = $this->idMapper->getMapping('users', $row->pilotid);
$airline_id = $this->idMapper->getMapping('airlines', $row->code);
@ -70,7 +91,10 @@ class PirepImporter extends BaseImporter
$attrs['level'] = 0;
}
$pirep = Pirep::updateOrCreate(['id' => $pirep_id], $attrs);
$w = ['id' => $pirep_id];
$pirep = Pirep::updateOrCreate($w, $attrs);
//$pirep = Pirep::create(array_merge($w, $attrs));
//Log::debug('pirep oldid='.$pirep_id.', olduserid='.$row->pilotid
// .'; new id='.$pirep->id.', user id='.$user_id);

4
modules/Installer/Services/Importer/Importers/RankImport.php
View File

@ -7,12 +7,14 @@ use Modules\Installer\Services\Importer\BaseImporter;
class RankImport extends BaseImporter
{
protected $table = 'ranks';
public function run($start = 0)
{
$this->comment('--- RANK IMPORT ---');
$count = 0;
foreach ($this->db->readRows('ranks') as $row) {
foreach ($this->db->readRows($this->table, $start) as $row) {
$rank = Rank::firstOrCreate(['name' => $row->rank], [
'image_url' => $row->rankimage,
'hours' => $row->minhours,

4
modules/Installer/Services/Importer/Importers/UserImport.php
View File

@ -13,6 +13,8 @@ use Modules\Installer\Services\Importer\BaseImporter;
class UserImport extends BaseImporter
{
protected $table = 'pilots';
/**
* @var UserService
*/
@ -26,7 +28,7 @@ class UserImport extends BaseImporter
$count = 0;
$first_row = true;
foreach ($this->db->readRows('pilots', $start) as $row) {
foreach ($this->db->readRows($this->table, $start) as $row) {
$pilot_id = $row->pilotid; // This isn't their actual ID
$name = $row->firstname.' '.$row->lastname;

15
modules/Installer/Services/Importer/Stages/Stage2.php
View File

@ -1,15 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\AirportImporter;
class Stage2 extends BaseStage
{
public $importers = [
AirportImporter::class,
];
public $nextStage = 'stage3';
}

15
modules/Installer/Services/Importer/Stages/Stage3.php
View File

@ -1,15 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\UserImport;
class Stage3 extends BaseStage
{
public $importers = [
UserImport::class,
];
public $nextStage = 'stage4';
}

15
modules/Installer/Services/Importer/Stages/Stage4.php
View File

@ -1,15 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\FlightImporter;
class Stage4 extends BaseStage
{
public $importers = [
FlightImporter::class,
];
public $nextStage = 'stage5';
}

15
modules/Installer/Services/Importer/Stages/Stage5.php
View File

@ -1,15 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\PirepImporter;
class Stage5 extends BaseStage
{
public $importers = [
PirepImporter::class,
];
public $nextStage = 'stage6';
}

71
modules/Installer/Utils/ImporterDB.php
View File

@ -3,8 +3,6 @@
namespace Modules\Installer\Utils;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use PDO;
use PDOException;
@ -13,14 +11,25 @@ use PDOException;
*/
class ImporterDB
{
/**
* @var int
*/
public $batchSize;
/**
* @var PDO
*/
private $conn;
/**
* @var string
*/
private $dsn;
/**
* @var array
*/
private $creds;
private $batchSize;
public function __construct($creds)
{
@ -33,7 +42,12 @@ class ImporterDB
Log::info('Using DSN: '.$this->dsn);
$this->batchSize = config('installer.importer.batch_size');
$this->batchSize = config('installer.importer.batch_size', 20);
}
public function __destruct()
{
$this->close();
}
public function connect()
@ -43,7 +57,15 @@ class ImporterDB
$this->conn->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ);
} catch (PDOException $e) {
Log::error($e);
exit();
throw $e;
}
}
public function close()
{
if ($this->conn) {
$this->conn = null;
}
}
@ -85,13 +107,11 @@ class ImporterDB
*
* @param string $table The name of the table
* @param int [$start_offset]
*
* @throws \Modules\Installer\Exceptions\ImporterNextRecordSet
* @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
* @param string [$fields]
*
* @return \Generator
*/
public function readRows($table, $start_offset = 0)
public function readRows($table, $start_offset = 0, $fields = '*')
{
$this->connect();
@ -104,8 +124,8 @@ class ImporterDB
$rows_to_read = $total_rows;
}
Log::info('Reading '.$offset.' to '.$rows_to_read.' of '.$total_rows);
yield from $this->readRowsOffset($table, $this->batchSize, $offset);
// Log::info('Reading '.$offset.' to '.$rows_to_read.' of '.$total_rows);
yield from $this->readRowsOffset($table, $this->batchSize, $offset, $fields);
$offset += $this->batchSize;
}
@ -115,38 +135,27 @@ class ImporterDB
* @param string $table
* @param int $limit Number of rows to read
* @param int $offset Where to start from
*
* @throws ImporterNextRecordSet
* @throws ImporterNoMoreRecords
* @param string [$fields]
*
* @return \Generator
*/
public function readRowsOffset($table, $limit, $offset)
public function readRowsOffset($table, $limit, $offset, $fields = '*')
{
$sql = 'SELECT * FROM '.$this->tableName($table).' LIMIT '.$limit.' OFFSET '.$offset;
if (is_array($fields)) {
$fields = implode(',', $fields);
}
$sql = 'SELECT '.$fields.' FROM '.$this->tableName($table).' LIMIT '.$limit.' OFFSET '.$offset;
try {
$result = $this->conn->query($sql);
if (!$result) {
throw new ImporterNoMoreRecords();
}
$rowCount = $result->rowCount();
if (!$rowCount === 0) {
throw new ImporterNoMoreRecords();
if (!$result || $result->rowCount() === 0) {
return;
}
foreach ($result as $row) {
yield $row;
}
// No more records left since we got the number below the limit
if ($rowCount < $limit) {
throw new ImporterNoMoreRecords();
}
throw new ImporterNextRecordSet($offset + $limit);
} catch (PDOException $e) {
// Without incrementing the offset, it should re-run the same query
Log::error('Error readRowsOffset: '.$e->getMessage());